├── img
    ├── boxed.png
    └── learning.mp4
├── samples
    ├── test1.jpg
    └── tensor-convo-20230122-final-gelu.state
├── .gitignore
├── basic-autograd.cc
├── vizi.hh
├── textsupport.hh
├── model.hh
├── LICENSE
├── los3.cc
├── try-convo.cc
├── tensormodtest.cc
├── imagine.cc
├── .github
    └── workflows
    │   └── cmake.yml
├── mnistreader.hh
├── mnistposter.cc
├── grutest.cc
├── textsupport.cc
├── convo-alphabet.hh
├── misc.hh
├── mnistreader.cc
├── cnn1.hh
├── 37learn.cc
├── vizi.cc
├── gru-layer.hh
├── ext
    └── sqlitewriter
    │   ├── sqlwriter.hh
    │   └── sqlwriter.cc
├── threeorseven.cc
├── CMakeLists.txt
├── cnn-alphabet.hh
├── testfvector.cc
├── fvector.hh
├── tensor-relu.cc
├── alternate
    ├── tensor.cc
    ├── modular-convo.cc
    ├── first-relu.cc
    ├── modular-linear-convo.cc
    ├── workertest.cc
    ├── modular-threaded-convo.cc
    ├── worker-convo-avx-threaded.cc
    ├── worker-convo.cc
    └── worker-convo-avx.cc
├── tensor-convo.cc
├── trackedfuncs.hh
├── tensor-gru.cc
├── ui.cc
├── gru.hh
├── los.cc
├── los2.cc
├── testrunner.cc
├── hello-dl.cc
├── layers.hh
└── tensor-layers.hh


/img/boxed.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/berthubert/hello-dl/main/img/boxed.png


--------------------------------------------------------------------------------
/img/learning.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/berthubert/hello-dl/main/img/learning.mp4


--------------------------------------------------------------------------------
/samples/test1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/berthubert/hello-dl/main/samples/test1.jpg


--------------------------------------------------------------------------------
/samples/tensor-convo-20230122-final-gelu.state:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/berthubert/hello-dl/main/samples/tensor-convo-20230122-final-gelu.state


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | *~
 2 | hello-dl
 3 | # Prerequisites
 4 | *.d
 5 | 
 6 | # Compiled Object files
 7 | *.slo
 8 | *.lo
 9 | *.o
10 | *.obj
11 | 
12 | # Precompiled Headers
13 | *.gch
14 | *.pch
15 | 
16 | # Compiled Dynamic libraries
17 | *.so
18 | *.dylib
19 | *.dll
20 | 
21 | # Fortran module files
22 | *.mod
23 | *.smod
24 | 
25 | # Compiled Static libraries
26 | *.lai
27 | *.la
28 | *.a
29 | *.lib
30 | 
31 | # Executables
32 | *.exe
33 | *.out
34 | *.app
35 | 


--------------------------------------------------------------------------------
/basic-autograd.cc:
--------------------------------------------------------------------------------
 1 | #include "tensor2.hh"
 2 | #include <iostream>
 3 | using namespace std;
 4 | 
 5 | int main()
 6 | {
 7 | #if 0
 8 |   Tensor x(2.0f);
 9 |   Tensor z(0.0f);
10 |   Tensor a(1.0f);
11 |   Tensor y = x * (z + a);
12 |   y(0,0);
13 |   y.backward();
14 |   
15 | #else 
16 |   Tensor x(2.0f);
17 |   Tensor z(0.0f);
18 |   Tensor y = Tensor(3.0f)*x*x*x + Tensor(4.0f)*x + Tensor(1.0f) + x*z;
19 |   y(0,0);
20 |   y.backward();  
21 |   cout << "y = "<< y << endl; // 3*8 + 4*2 + 1 = 33
22 |   
23 | 
24 | 
25 |   cout << "dy/dx = " << x.getGrad() << endl; // 9*x^2 + 4 = 40
26 |   cout << "dy/dz = " << z.getGrad() << endl; // 2
27 |   #endif
28 | }
29 | 


--------------------------------------------------------------------------------
/vizi.hh:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <iostream>
 3 | #include <functional>
 4 | #include "tensor2.hh"
 5 | #include "ext/stb/stb_truetype.h"
 6 | 
 7 | template<typename T>
 8 | void printImgTensor(const T& img)
 9 | {
10 |   for(unsigned int y=0; y < img.getRows(); ++y) {
11 |     for(unsigned int x=0; x < img.getCols(); ++x) {
12 |       float val = img(y,x);
13 |       if(val > 0.5)
14 |         std::cout<<'X';
15 |       else if(val > 0.25)
16 |         std::cout<<'*';
17 |       else if(val > 0.125)
18 |         std::cout<<'.';
19 |       else
20 |         std::cout<<' ';
21 |     }
22 |     std::cout<<'\n';
23 |   }
24 |   std::cout<<"\n";
25 | }
26 | 
27 | struct FontWriter
28 | {
29 |   FontWriter();
30 |   void writeChar(char ch, int s, int c, int r, std::function<void(int, int, int, int, int)> f);
31 |   stbtt_fontinfo d_font;
32 |   std::vector<char> d_ttf_buffer; 
33 | };
34 | 
35 | void saveTensor(const Tensor<float>& t, const std::string& fname, int size, bool monochrome=false);
36 | 


--------------------------------------------------------------------------------
/textsupport.hh:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <unordered_map>
 3 | #include <stdexcept>
 4 | #include <vector>
 5 | #include <string>
 6 | #include <iostream>
 7 | std::vector<std::string> textChopper(const char* fname, size_t siz, int mult=1);
 8 | 
 9 | class BiMapper
10 | {
11 | public:
12 |   explicit BiMapper(const char* fname, int lim=-1);
13 |   int c2i(char c) const
14 |   {
15 |     auto iter = d_c2i.find(c);
16 |     if(iter == d_c2i.end()) {
17 |       //      std::cout<<("Attempting to find unknown character with value '"+std::to_string((int)c)+"'")<<std::endl;
18 |       return 0;
19 |     }
20 |     return iter->second;
21 |   }
22 |   char i2c(int i) const
23 |   {
24 |     auto iter = d_i2c.find(i);
25 |     if(iter == d_i2c.end()) {
26 |       //      std::cout<<("Attempting to find unknown integer "+std::to_string(i))<<std::endl;
27 |       return '?';
28 |     }
29 |     return iter->second;
30 |   }
31 | 
32 | private:
33 |   std::unordered_map<int, char> d_c2i;
34 |   std::unordered_map<char, int> d_i2c;
35 | };
36 | 
37 | 
38 | 


--------------------------------------------------------------------------------
/model.hh:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include "layers.hh"
 3 | #include <sstream>
 4 | 
 5 | struct ModelState
 6 | {
 7 |   std::vector<LayerBase*> d_members;
 8 |   void save(std::ostream& out) const
 9 |   {
10 |     for(const auto& mem : d_members)
11 |       mem->save(out);
12 |   }
13 |   void save(std::string& out) const
14 |   {
15 |     std::ostringstream os;
16 |     for(const auto& mem : d_members)
17 |       mem->save(os);
18 | 
19 |     out=os.str();
20 |   }
21 |   
22 |   void load(std::istream& in)
23 |   {
24 |     for(auto& mem : d_members)
25 |       mem->load(in);
26 |   }
27 | 
28 |   void load(std::string& in)
29 |   {
30 |     std::istringstream is(in);
31 |     load(is);
32 |   }
33 | 
34 |   void learn(float lr)
35 |   {
36 |     for(auto& mem : d_members)
37 |       mem->learn(lr);
38 |   }
39 |   
40 |   void zeroGrad()
41 |   {
42 |     for(auto& mem : d_members)
43 |       mem->zeroGrad();
44 |   }
45 | 
46 |   uint32_t size()
47 |   {
48 |     size_t ret = 0;
49 |     for(auto& mem : d_members)
50 |       ret += mem->size();
51 |     return ret;
52 |   }
53 |   
54 | };
55 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2022 bert hubert
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/los3.cc:
--------------------------------------------------------------------------------
 1 | #define EIGEN_USE_THREADS
 2 | #include <iostream>
 3 | #include <stdlib.h>
 4 | #include "tensor2.hh"
 5 | #include "misc.hh"
 6 | 
 7 | #include <Eigen/CXX11/Tensor>
 8 | using namespace std;
 9 | 
10 | int main(int argc, char**argv)
11 | {
12 |   DTime dt;
13 |   dt.start();
14 |   float t=0;
15 |   for(unsigned int n=0; n < 10000/64; ++n) {
16 |     Eigen::Tensor<float, 3> input(64,28, 28);
17 |     Eigen::Tensor<float, 2> kernel(3, 3);
18 |     Eigen::Tensor<float, 3> output(64, 26, 26);
19 |     input.setRandom();
20 |     kernel.setRandom();
21 |     
22 |     Eigen::array<ptrdiff_t, 2> dims({1, 2});  // Specify second and third dimension for convolution.
23 |     output = input.convolve(kernel, dims);
24 |     t+=output(0,0,0);
25 |   }
26 |   cout<<"t: "<<t<<", "<<dt.lapUsec() << endl;
27 |   t=0;
28 |   dt.start();
29 |   for(unsigned int n =0 ; n <10000; ++n) {
30 |     Tensor i(28,28);
31 |     Tensor k(3,3);
32 |     Tensor b(1,1);
33 |     i.randomize(1);
34 |     k.randomize(1);
35 |     b.randomize(1);
36 |     Tensor c=i.makeConvo(3, k, b);
37 | 
38 |     t += c(0,0);
39 |   }
40 |   cout<<"t: "<<t<<", "<<dt.lapUsec() << endl;
41 |   
42 | }
43 | 


--------------------------------------------------------------------------------
/try-convo.cc:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <memory>
 3 | #include "mnistreader.hh"
 4 | #include "misc.hh"
 5 | #include <string.h>
 6 | #include <fenv.h>
 7 | #include "tensor-layers.hh"
 8 | #include "vizi.hh"
 9 | 
10 | using namespace std;
11 | 
12 | int main(int argc, char** argv)
13 | {
14 |   if(argc <  2) {
15 |     cerr<<"Syntax: try-convo index"<<endl;
16 |     return 0;
17 |   }
18 | 
19 |   feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
20 | 
21 |   MNISTReader mntest("gzip/emnist-letters-test-images-idx3-ubyte.gz", "gzip/emnist-letters-test-labels-idx1-ubyte.gz");
22 |   cout<<"Have "<<mntest.num()<<" validation images"<<endl;
23 | 
24 |   Tensor img(28, 28);
25 |   
26 |   int idx = atoi(argv[1]);
27 |   mntest.pushImage(idx, img);
28 | 
29 |   Conv2d<float, 28, 28, 3, 1, 1> convo;
30 |   auto& f1 = convo.d_filters[0];
31 |   f1(0,0) = -1;  f1(0,1) = -1; f1(0,2)=1;
32 |   f1(1,0) = -1;  f1(0,1) = -1; f1(1,2)=1;
33 |   f1(2,0) =  1;  f1(2,1) =  1; f1(2,2)=1;
34 | 
35 |   convo.d_bias[0](0,0) = 0;
36 | 
37 |   Tensor out = convo.forward(img)[0].makeMax2d(2);
38 |   out(0,0);
39 |   cout<<"out:\n"<<out<<endl;
40 |   saveTensor(img, "input.png", 252, true);
41 |   saveTensor(out, "convolved.png", 9*out.getRows(), false);
42 | }
43 | 


--------------------------------------------------------------------------------
/tensormodtest.cc:
--------------------------------------------------------------------------------
 1 | #include "ext/doctest.h"
 2 | #include "tensor2.hh"
 3 | #include <iostream>
 4 | #include <Eigen/Dense>
 5 | #include "misc.hh"
 6 | #include <fstream>
 7 | #include <unistd.h>
 8 | #include <fcntl.h>
 9 | #include "tensor-layers.hh"
10 | using namespace std;
11 | 
12 | 
13 | TEST_CASE("tensor model load save test") {
14 |   struct TestState : ModelState<float>
15 |   {
16 |     Conv2d<float, 28, 28, 3, 1,  32> d_lc1; // -> 26*26 -> max2d -> 13*13
17 |     Linear<float, 512, 64> d_fc1;  
18 |     TestState()
19 |     {
20 |       d_members = {{&d_lc1, "lc1"}, {&d_fc1, "fc1"}};
21 |     }
22 |   };
23 | 
24 |   TestState ts;
25 |   ts.randomize();
26 |   
27 |   saveModelState(ts, "tensormodetest-test.state");
28 | 
29 |   TestState ts2;
30 |   ts2.randomize(); // just to confuse things
31 |   loadModelState(ts2, "tensormodetest-test.state");
32 | 
33 |   CHECK(ts.d_lc1.d_filters[16](2,2) == ts2.d_lc1.d_filters[16](2,2));
34 |   CHECK(ts.d_fc1.d_weights(30,32) == ts2.d_fc1.d_weights(30,32));
35 | }
36 | 
37 | 
38 | TEST_CASE("max2d array") {
39 |   std::array<Tensor<float>, 3> in;
40 |   for(unsigned int n=0; n < in.size(); ++n) {
41 |     in[n]=Tensor<float>(4,4);
42 |     in[n].iota(n);
43 |   }
44 |   auto res = Max2dfw(in, 2);
45 |   // 0   1  2  3
46 |   // 4   5  6  7
47 |   // 8   9 10 11
48 |   // 12 13 14 15
49 | 
50 |   CHECK(res[0](0,0) == 5);
51 |   CHECK(res[0](0,1) == 7);
52 | 
53 |   CHECK(res[1](0,0) == 6);
54 |   CHECK(res[1](0,1) == 8);
55 | 
56 |   CHECK(res[2](1,0) == 15);
57 |   CHECK(res[2](1,1) == 17);
58 | }
59 | 
60 | 


--------------------------------------------------------------------------------
/imagine.cc:
--------------------------------------------------------------------------------
 1 | #include <array>
 2 | #include <vector>
 3 | #include <fstream>
 4 | #include <iostream>
 5 | #include "tensor-layers.hh"
 6 | #include "convo-alphabet.hh"
 7 | #include "vizi.hh"
 8 | #include "mnistreader.hh"
 9 | using namespace std;
10 | 
11 | 
12 | int main(int argc, char** argv)
13 | {
14 |   if(argc < 3) {
15 |     cout<<"Syntax: imagine fromletter toletter modelname"<<endl;
16 |     return EXIT_FAILURE;
17 |   }
18 | 
19 |   int fromlabel = *argv[1]-'a' + 1;
20 |   int tolabel = *argv[2]-'a' + 1;
21 |   
22 |   ConvoAlphabetModel m;
23 |   ConvoAlphabetModel::State s;
24 |   cout<<"Loading model state from file '"<<argv[3]<<"'\n";
25 |   loadModelState(s, argv[3]);
26 |   m.init(s, true);
27 | 
28 |   MNISTReader mntest("gzip/emnist-letters-test-images-idx3-ubyte.gz", "gzip/emnist-letters-test-labels-idx1-ubyte.gz");
29 |   m.img.zero();
30 |   for(int n = 0 ; n < mntest.num(); ++n) {
31 |     if(mntest.getLabel(n) == fromlabel) {
32 |       mntest.pushImage(n, m.img);
33 |       break;
34 |     }
35 |   }
36 |   /*
37 |   m.img.randomize(1.0);
38 |   m.img.d_imp->d_val = m.img.d_imp -> d_val.unaryExpr([](float v) { return fabs(v); });
39 |   */
40 |   
41 |   m.img.normalize(0.172575, 0.25);
42 | 
43 |   auto specscore = m.scores.makeSlice(tolabel, 0, 1, 1);
44 |   auto topo = specscore.getTopo();
45 |   for(unsigned int tries = 0 ; tries < 10000; ++tries) {
46 |     cout<<specscore<<endl;
47 |     specscore.backward(topo);
48 |     auto grad = m.img.getGrad();
49 |     grad *= 0.2;
50 |     m.img.d_imp->d_val += grad;
51 | 
52 |     if(!(tries %4))
53 |       saveTensor(m.img, "imagine-"+to_string(tries)+".png", 252, true);
54 |     specscore.zerograd(topo);
55 |   }
56 |   
57 | }
58 | 


--------------------------------------------------------------------------------
/.github/workflows/cmake.yml:
--------------------------------------------------------------------------------
 1 | name: CMake
 2 | 
 3 | on:
 4 |   push:
 5 |     branches: [ main ]
 6 |   pull_request:
 7 |     branches: [ main ]
 8 | 
 9 | env:
10 |   # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
11 |   BUILD_TYPE: Release
12 | 
13 | 
14 | jobs:
15 |   build:
16 |     # The CMake configure and build commands are platform agnostic and should work equally
17 |     # well on Windows or Mac.  You can convert this to a matrix build if you need
18 |     # cross-platform coverage.
19 |     # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
20 |     runs-on: ubuntu-latest
21 | 
22 |     steps:
23 |     - uses: actions/checkout@v3
24 | 
25 |     - name: Install libraries
26 |       run: sudo apt-get install libeigen3-dev libz-dev libsqlite3-dev
27 | 
28 |     - name: Configure CMake
29 |       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
30 |       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
31 |       run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
32 | 
33 |     - name: Build
34 |       # Build your program with the given configuration
35 |       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
36 | 
37 |     - name: Test
38 |       working-directory: ${{github.workspace}}/build
39 |       # Execute tests defined by the CMake configuration.  
40 |       # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
41 |       run: ctest -C ${{env.BUILD_TYPE}}
42 |       
43 | 


--------------------------------------------------------------------------------
/mnistreader.hh:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <string>
 3 | #include <vector>
 4 | #include <unordered_map>
 5 | //#include "array.hh"
 6 | //#include "fvector.hh"
 7 | 
 8 | #include "tensor2.hh"
 9 | class MNISTReader
10 | {
11 | public:
12 |   MNISTReader(const std::string& images, const std::string& labels);
13 |   unsigned int num() const
14 |   {
15 |     return d_num;
16 |   }
17 |   std::vector<uint8_t> getImage(int n) const;
18 |   const std::vector<float>& getImageFloat(int n) const
19 |   {
20 |     if(auto iter = d_converted.find(n); iter != d_converted.end())
21 |       return iter->second;
22 |     else
23 |       throw std::runtime_error("Could not find image "+std::to_string(n));
24 |   }
25 | 
26 |   template<typename T>
27 |   void pushImage(int n, Tensor<T>& dest) const
28 |   {
29 |     assert(dest.d_imp && dest.d_imp->d_mode == TMode::Parameter);
30 |     const auto& src = getImageFloat(n);
31 |     for(int row=0 ; row < 28; ++row)
32 |       for(int col=0 ; col < 28; ++col)
33 |         dest(row, col) = src.at(row+28*col);
34 |   }
35 |   /*
36 |   template<typename UT>
37 |   void pushImage(int n, NNArray<UT, 28, 28>& dest, int idx) const
38 |   {
39 |     const auto& src = getImageFloat(n);
40 |     for(int row=0 ; row < 28; ++row)
41 |       for(int col=0 ; col < 28; ++col) {
42 |         if(!dest(row,col).impl) // XXX FUGLY
43 |           dest(row, col) = 0;
44 |         dest(row, col).impl->d_val.v[idx] = src.at(row+28*col);
45 |       }
46 |   }
47 |   */
48 |   
49 |   char getLabel(int n) const;
50 | private:
51 |   std::vector<uint8_t> d_images;
52 |   std::vector<char> d_labels;
53 |   unsigned int d_rows, d_cols, d_stride, d_num;
54 |   std::unordered_map<int, std::vector<float>> d_converted;
55 | };
56 | 


--------------------------------------------------------------------------------
/mnistposter.cc:
--------------------------------------------------------------------------------
 1 | 
 2 | #define STB_IMAGE_WRITE_IMPLEMENTATION
 3 | #include "ext/stb/stb_image_write.h"
 4 | 
 5 | #include <array>
 6 | #include <vector>
 7 | #include <fstream>
 8 | #include <iostream>
 9 | #include "mnistreader.hh"
10 | #include <fenv.h>
11 | #include "misc.hh"
12 | 
13 | using namespace std;
14 | 
15 | int main(int argc, char **argv)
16 | {
17 |   int filt=-1;
18 |   if(argc == 2)
19 |     filt= 1 + argv[1][0] - 'a';
20 |   
21 |   feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
22 | 
23 |   MNISTReader mn("gzip/emnist-letters-train-images-idx3-ubyte.gz", "gzip/emnist-letters-train-labels-idx1-ubyte.gz");
24 |     //MNISTReader mn("gzip/emnist-letters-test-images-idx3-ubyte.gz", "gzip/emnist-letters-test-labels-idx1-ubyte.gz");
25 | 
26 |   cout<<"Have "<<mn.num()<<" images"<<endl;
27 | 
28 |   constexpr int imgrows=1200, imgcols=1900;
29 |   vector<uint8_t> out;
30 |   out.resize(imgcols*imgrows);
31 |   auto pix = [&out, &imgrows, &imgcols](int col, int row) -> uint8_t&
32 |   {
33 |     return out[col + row*imgcols];
34 |   };
35 |   
36 |   int count=0;
37 |   Batcher batcher(mn.num());
38 |   for(;;) {
39 |     auto b = batcher.getBatch(1);
40 |     if(b.empty())
41 |       break;
42 |     int n=b[0];
43 |     if(filt >=0 && mn.getLabel(n) != filt)
44 |         continue;
45 |     
46 |     Tensor img(28,28);
47 |     mn.pushImage(n, img);
48 |     
49 |     int x = 30 * (count % (imgcols/30 - 1)); // this many per row
50 |     int y = 30 * (count / (imgcols/30 - 1));
51 |     count++;
52 | 
53 |     if(x+30 >= imgcols || y+30 >= imgrows)
54 |       break;
55 |     
56 |     for(unsigned int r=0; r < img.getRows(); ++r)
57 |       for(unsigned int c=0; c < img.getCols(); ++c)
58 |         pix(x+c, y+r) = 255 - img(r,c)*255;
59 |   }
60 |   stbi_write_png("poster.png", imgcols, imgrows, 1, &out[0], imgcols);
61 | }
62 | 


--------------------------------------------------------------------------------
/grutest.cc:
--------------------------------------------------------------------------------
 1 | #include "ext/doctest.h"
 2 | #include "tracked.hh"
 3 | #include "gru.hh"
 4 | #include <iostream>
 5 | 
 6 | using namespace std;
 7 | 
 8 | // a sequence needs to be dragged through
 9 | #if 0
10 | TEST_CASE("single GRU") {
11 |   GRULayer<float, 26, 250> gm;
12 |   //  cout<<"gm.size(): "<<gm.size() << endl;
13 |   Linear<float, 250, 26> fc;
14 |   //  cout<<"fc.size(): "<<fc.size() << endl;
15 |   
16 |   NNArray<float, 26, 1> in;
17 |   NNArray<float, 1, 26> expected;
18 | 
19 |   std::string input = "hellothisisbert";
20 |   std::string output;
21 |   TrackedFloat totloss=0.0;
22 |   for(size_t pos = 0 ; pos < input.size() - 1; ++pos) {
23 |     in.zero();  in(input.at(pos)-'a', 0) = 1.0;
24 |     
25 |     expected.zero(); expected(0, input.at(pos+1)-'a') = 1.0;
26 |     auto res1 = fc.forward(gm.forward(in));
27 |     auto score = res1.logSoftMax();
28 |     auto loss = TrackedNumber<float>(0.0) - (expected*score)(0,0);
29 |     totloss = totloss + loss;
30 |     output.append(1, 'a' + score.maxValueIndexOfColumn(0));
31 |     //    cout<<"score: "<<loss.getVal() << endl;
32 |   }
33 |   //  cout<<output<<endl;
34 |   totloss.backward();
35 |   //  cout<<gm.d_w_ir.getGrad()<<endl;
36 | }
37 | 
38 | 
39 | TEST_CASE("multi GRU") {
40 |   GRULayer<float, 10, 250> gm1, gm2, gm3;
41 |   //  cout<<"gm.size(): "<<gm1.size() << endl;
42 |   Linear<float, 250, 10> fc1, fc2, fc3;
43 |   
44 |   NNArray<float, 10, 1> in1, in2, in3;
45 |   in1.zero();  in1(3,0) = 1.0;
46 |   in2.zero();  in2(1,0) = 1.0;
47 |   in3.zero();  in3(4,0) = 1.0;
48 |   auto res1 = gm1.forward(in1);
49 |   gm2.d_prevh = res1;
50 | 
51 |   auto res2 = gm2.forward(in2);
52 |   gm3.d_prevh = res2;
53 | 
54 |   auto res3 = gm3.forward(in3);
55 | 
56 |   cout<<"GRU: "<<endl;
57 |   cout << fc1.forward(res1).logSoftMax().maxValueIndexOfColumn(0) << endl;
58 |   cout << fc2.forward(res2).logSoftMax().maxValueIndexOfColumn(0) << endl;
59 |   cout << fc3.forward(res3).logSoftMax().maxValueIndexOfColumn(0) << endl;
60 |   
61 |   
62 | }
63 | #endif
64 | 


--------------------------------------------------------------------------------
/textsupport.cc:
--------------------------------------------------------------------------------
 1 | #include "textsupport.hh"
 2 | #include <iostream>
 3 | #include <array>
 4 | #include <fstream>
 5 | #include <vector>
 6 | #include <map>
 7 | #include <algorithm>
 8 | #include <random>
 9 | 
10 | using namespace std;
11 | 
12 | vector<string> textChopper(const char* fname, size_t siz, int mult)
13 | {
14 |   ifstream ifs(fname);
15 | 
16 |   vector<char> buffer(1024000);
17 |   string total;
18 |   while(!ifs.eof()) {
19 |     ifs.read(&buffer[0], buffer.size());
20 |     total.append(&buffer[0], &buffer[ifs.gcount()]);
21 |   }
22 |   buffer.clear();
23 |   unsigned int pieces = mult*total.size()/siz;
24 |   vector<string> ret;
25 |   ret.reserve(pieces);
26 | 
27 |   std::random_device rd;  //Will be used to obtain a seed for the random number engine
28 |   std::mt19937 gen(rd()); //Standard mersenne_twister_engine seeded with rd()
29 |   std::uniform_int_distribution<> distrib(0, total.size() - siz -1);
30 |   
31 |   for(unsigned int n = 0 ; n < pieces; ++n) {
32 |     ret.push_back(total.substr(distrib(gen), siz));
33 |     for(auto& c : *ret.rbegin())
34 |       if(c=='\n' || c=='\t') c=' ';
35 |         
36 |   }
37 |   
38 |   return ret;
39 | }
40 | 
41 | 
42 | BiMapper::BiMapper(const char* fname, int lim)
43 | {
44 |   ifstream ifs(fname);
45 |   std::array<unsigned char, 4096> a;
46 |   std::unordered_map<int, int> popcount;
47 |   while(!ifs.eof()) {
48 |     ifs.read((char*)&a[0], a.size());
49 |     for(const auto& c : a) {
50 |       //cout<<c;
51 |       if(c<127)
52 |         popcount[c]++;
53 |     }
54 |   }
55 |   vector<pair<int,int>> revcount;
56 |   for(const auto& p : popcount)
57 |     revcount.push_back(p);
58 |   sort(revcount.begin(), revcount.end(), [](const auto& a, const auto& b) {
59 |     return b.second < a.second;
60 |   });
61 | 
62 |   if(lim >= 0 && revcount.size() > (unsigned int)lim)
63 |     revcount.resize(lim);
64 |   
65 |   for(unsigned int n=0; n < revcount.size(); ++n) {
66 |     d_c2i[revcount[n].first] = n;
67 |     d_i2c[n]=revcount[n].first;
68 |     //    cout<<(char)revcount[n].first <<" -> "<<n<<"\n";
69 |   }
70 |   cout<<"Assigned "<<d_c2i.size()<<" mappings"<<endl;
71 | }
72 | 
73 | 


--------------------------------------------------------------------------------
/convo-alphabet.hh:
--------------------------------------------------------------------------------
 1 | #include "tensor-layers.hh"
 2 | // Model from https://data-flair.training/blogs/handwritten-character-recognition-neural-network/
 3 | struct ConvoAlphabetModel {
 4 |   Tensor<float> img{28,28};
 5 |   Tensor<float> scores{26, 1};
 6 |   Tensor<float> expected{1,26};
 7 |   Tensor<float> modelloss{1,1};
 8 |   Tensor<float> weightsloss{1,1};
 9 |   Tensor<float> loss{1,1};
10 | 
11 |   struct State : public ModelState<float>
12 |   {
13 |     //           r_in c   k c_i  c_out
14 |     Conv2d<float, 28, 28, 3, 1,  32> c1; // -> 26*26 -> max2d -> 13*13
15 |     Conv2d<float, 13, 13, 3, 32, 64> c2; // -> -> 11*11 -> max2d -> 6*6 //padding
16 |     Conv2d<float, 6,   6, 3, 64, 128> c3; // -> 4*4 -> max2d -> 2*2
17 |     // flattened to 512 (128*2*2)
18 |            //      IN OUT
19 |     Linear<float, 512, 64> fc1;  
20 |     Linear<float, 64, 128> fc2;
21 |     Linear<float, 128, 26> fc3; 
22 | 
23 |     State()
24 |     {
25 |       this->d_members = {{&c1, "c1"}, {&c2, "c2"}, {&c3, "c3"}, {&fc1, "fc1"}, {&fc2, "fc2"}, {&fc3, "fc3"}};
26 |     }
27 |   };
28 |   
29 |   void init(State& s, bool production=false)
30 |   {
31 |     using ActFunc = GeluFunc;
32 | 
33 |     img.zero();
34 |     img.d_imp->d_nograd=true;
35 |     
36 |     auto step1 = s.c1.forward(img);    // -> 26x26, 32 layers
37 |     auto step2 = Max2dfw(step1, 2);    // -> 13x13
38 |     auto step3 = s.c2.forward(step2);  // -> 11x11, 64 layers
39 |     auto step4 = Max2dfw(step3, 2);    // -> 6x6 (padding)
40 |     auto step5 = s.c3.forward(step4);  // -> 4x4, 128 layers
41 |     auto step6 = Max2dfw(step5, 2);    // -> 2x2
42 |     auto flat = makeFlatten(step6);    // -> 512x1
43 |     auto output = s.fc1.forward(flat); // -> 64
44 |     auto output1 = production ? output : output.makeDropout(0.5); 
45 |     auto output2 = makeFunction<ActFunc>(output1);
46 |     auto output3 = makeFunction<ActFunc>(s.fc2.forward(output2)); // -> 128
47 |     //    auto output4 = makeFunction<ActFunc>(s.fc3.forward(output3)); // -> 26
48 |     auto output4 = s.fc3.forward(output3); // -> 26
49 |     scores = makeLogSoftMax(output4);
50 |     modelloss = -(expected*scores);
51 | 
52 |     Tensor<float> fact(1,1);
53 |     fact(0,0) = 0.02;
54 |     weightsloss = fact*(s.c1.SquaredWeightsSum() +  s.c2.SquaredWeightsSum() +  s.c3.SquaredWeightsSum() +
55 |                         s.fc1.SquaredWeightsSum() + s.fc1.SquaredWeightsSum() + s.fc1.SquaredWeightsSum());
56 | 
57 |     loss = modelloss; // + weightsloss;
58 |   }
59 | };
60 | 


--------------------------------------------------------------------------------
/misc.hh:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include <iostream>
  3 | #include <random>
  4 | #include <algorithm>
  5 | #include <deque>
  6 | #include <vector>
  7 | #include <chrono>
  8 | #include <mutex>
  9 | #include <memory>
 10 | #include <atomic>
 11 | #include <optional>
 12 | 
 13 | struct HyperParameters
 14 | {
 15 |   float lr;
 16 |   float momentum;
 17 |   int batchMult;
 18 |   unsigned int getBatchSize()
 19 |   {
 20 |     return 8*batchMult;
 21 |   }
 22 | };
 23 | 
 24 | struct TrainingProgress
 25 | {
 26 |   int batchno=0;
 27 |   float lastTook=0;
 28 |   std::vector<float> losses;
 29 |   std::vector<float> corrects;
 30 |   std::atomic<unsigned int> trained=0;
 31 | };
 32 | 
 33 | extern struct TrainingProgress g_progress;
 34 | extern std::shared_ptr<HyperParameters> g_hyper;
 35 | int graphicsThread();
 36 | 
 37 | class Batcher
 38 | {
 39 | public:
 40 |   explicit Batcher(int n, std::optional<std::mt19937> rng=std::optional<std::mt19937>())
 41 |   {
 42 |     for(int i=0; i < n ; ++i)
 43 |       d_store.push_back(i);
 44 | 
 45 |     randomize(rng);
 46 |   }
 47 | 
 48 |   explicit Batcher(const std::vector<int>& in)
 49 |   {
 50 |     for(const auto& i : in)
 51 |       d_store.push_back(i);
 52 |     randomize();
 53 |   }
 54 | 
 55 |   auto getBatch(int n)
 56 |   {
 57 |     std::deque<int> ret;
 58 |     for(int i = 0 ; !d_store.empty() && i < n; ++i) {
 59 |       ret.push_back(d_store.front());
 60 |       d_store.pop_front();
 61 |     }
 62 |     return ret;
 63 |   }
 64 | 
 65 |   auto getBatchLocked(int n)
 66 |   {
 67 |     std::deque<int> ret;
 68 |     std::lock_guard<std::mutex> l(d_mut);
 69 |     for(int i = 0 ; !d_store.empty() && i < n; ++i) {
 70 |       ret.push_back(d_store.front());
 71 |       d_store.pop_front();
 72 |     }
 73 |     return ret;
 74 |   }
 75 | 
 76 | private:
 77 |   std::deque<int> d_store;
 78 |   std::mutex d_mut;
 79 |   void randomize(std::optional<std::mt19937> rnd = std::optional<std::mt19937>())
 80 |   {
 81 |     if(rnd) {
 82 |       std::shuffle(d_store.begin(), d_store.end(), *rnd);
 83 |     }
 84 |     else {
 85 |       std::random_device rd;
 86 |       std::mt19937 g(rd());
 87 |       std::shuffle(d_store.begin(), d_store.end(), g);
 88 |     }
 89 |   }
 90 | 
 91 | };
 92 | 
 93 | 
 94 | struct DTime
 95 | {
 96 |   void start()
 97 |   {
 98 |     d_start =   std::chrono::steady_clock::now();
 99 |   }
100 |   uint32_t lapUsec()
101 |   {
102 |     auto usec = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now()- d_start).count();
103 |     start();
104 |     return usec;
105 |   }
106 | 
107 |   std::chrono::time_point<std::chrono::steady_clock> d_start;
108 | };
109 | 


--------------------------------------------------------------------------------
/mnistreader.cc:
--------------------------------------------------------------------------------
 1 | #include "mnistreader.hh"
 2 | #include <string.h>
 3 | #include <utility>
 4 | #include <memory>
 5 | #include <stdexcept>
 6 | #include "zlib.h"
 7 | #include <arpa/inet.h>
 8 | 
 9 | using namespace std;
10 | 
11 | static auto safeOpen(const std::string& fname)
12 | {
13 |   gzFile fp = gzopen(fname.c_str(), "rb");
14 |   if(!fp)
15 |     throw runtime_error("Error opening file "+fname+": "+strerror(errno));
16 |   return fp;
17 | }
18 | 
19 | MNISTReader::MNISTReader(const std::string& images, const std::string& labels)
20 | {
21 |   struct idx1header
22 |   {
23 |     uint32_t magic;
24 |     uint32_t num;
25 |   } __attribute__((packed));
26 | 
27 |   struct idx3header
28 |   {
29 |     uint32_t magic;
30 |     uint32_t num;
31 |     uint32_t rows;
32 |     uint32_t cols;
33 |   } __attribute__((packed));
34 | 
35 |   auto imgfp = safeOpen(images);
36 |   auto labelsfp = safeOpen(labels);
37 | 
38 |   idx1header i1h;
39 |   idx3header i3h;
40 |   if(gzfread(&i1h, sizeof(idx1header), 1, labelsfp) != 1)
41 |     throw std::runtime_error("Label file too short");
42 |   if(gzfread(&i3h, sizeof(idx3header), 1, imgfp) != 1)
43 |     throw std::runtime_error("Images file too short");
44 | 
45 |   i1h.magic = htonl(i1h.magic);
46 |   i1h.num = htonl(i1h.num);
47 | 
48 |   i3h.magic = htonl(i3h.magic);
49 |   i3h.num = htonl(i3h.num);
50 |   i3h.rows = htonl(i3h.rows);
51 |   i3h.cols = htonl(i3h.cols);
52 | 
53 |   d_rows = i3h.rows;
54 |   d_cols = i3h.cols;
55 |   d_stride = d_rows * d_cols;
56 |   d_num = i3h.num;
57 |   if(i1h.magic != 2049)
58 |     throw runtime_error("Magic value of labels file wrong "+to_string(i1h.magic));
59 |   if(i3h.magic != 2051)
60 |     throw runtime_error("Magic value of images file wrong "+to_string(i3h.magic));
61 | 
62 |   if(i3h.num != i1h.num)
63 |     throw runtime_error("Mismatch between number of labels and number of images");
64 |   
65 |   d_images.resize(i3h.num*i3h.cols*i3h.rows);
66 |   if(gzfread((char*)&d_images[0], i3h.cols*i3h.rows, i3h.num, imgfp) != i3h.num)
67 |     throw runtime_error("Could not read all "+to_string(i3h.num)+" images");
68 | 
69 |   d_labels.resize(i3h.num);
70 |   if(gzfread((char*)&d_labels[0], 1, i3h.num, labelsfp) != i3h.num)
71 |     throw runtime_error("Could not read all "+to_string(i3h.num)+" labels");
72 | 
73 |   gzclose(imgfp);
74 |   gzclose(labelsfp);
75 | 
76 |   vector<float> tmp(28*28);
77 |   for(unsigned int n=0 ; n < d_num; ++n) {
78 |     unsigned int pos = n * d_stride;
79 |     for(unsigned int i=0; i < d_stride; ++i) {
80 |       tmp.at(i) = d_images.at(pos+i)/256.0;
81 |     }
82 |     d_converted[n]=tmp;
83 |   }
84 | }
85 | 
86 | vector<uint8_t> MNISTReader::getImage(int n) const
87 | {
88 |   unsigned int pos = n*d_rows*d_cols;
89 |   vector<uint8_t> ret(&d_images.at(pos), &d_images.at(pos + d_rows*d_cols));
90 |   return ret;
91 | }
92 | 
93 | char MNISTReader::getLabel(int n) const
94 | {
95 |   return d_labels.at(n);
96 | }
97 | 
98 | 


--------------------------------------------------------------------------------
/cnn1.hh:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include "tracked.hh"
 3 | #include "layers.hh"
 4 | 
 5 | /*
 6 | Gratefully copied from 'mnist.cpp' in the PyTorch example repository
 7 | https://github.com/pytorch/examples/blob/main/cpp/mnist/mnist.cpp
 8 | 
 9 | This model takes MNIST 28*28 input and:
10 | 
11 |   * normalizes to "0.1307, 03081", torch::data::transforms::Normalize<>(0.1307, 0.3081)
12 | 
13 |   * applies a 5*5 kernel convolution `conv1`, configured to emit 10 layers, 24*24
14 |   * does max_pool2d on these, which takes non-overlapping 2*2 rectangles 
15 |     and emits max value per rectangle. Delivers 12*12 values for each layer 
16 |   * ReLu
17 |   * does another 5x5 convolution `conv2` on the 10 layers, turning them into 20 layers of 8*8
18 |   * randomly *zeroes* half of the 20 layers `conv2_drop` - no state, Bernoulli 
19 |     STILL MISSING!
20 |   * another max_pool2d, 4*4*20 layers
21 |   * ReLu
22 |   * flatten to 320 values
23 |   * linear combination 320x50 (fc1)
24 |   * ReLU
25 |   * zero out half of values randomly during training (STILL MISSING)
26 |   * another linear combination, 50x10 (fc2)
27 |   * log_softmax on the 10 values
28 |   * the 10 outputs are probabilities per digit
29 |   * highest probability is chosen
30 |  */
31 | 
32 | 
33 | struct CNNModel {
34 |   NNArray<float, 28, 28> img;
35 | 
36 |   int label;
37 |   NNArray<float, 10, 1> scores;
38 |   NNArray<float, 1, 10> expected;
39 | 
40 |   TrackedFloat loss;
41 | 
42 |   struct State
43 |   {
44 |     //      R   C   K IN  OUTLAYERS
45 |     Conv2d<float, 28, 28, 5, 1,  10> c1; // -> 24*24 -> max2d -> 12*12
46 |     Conv2d<float, 12, 12, 5, 10, 20> c2; // -> 8*8 -> max2d -> 4*4
47 | 
48 |     //      IN OUT
49 |     Linear<float, 320, 50> fc1;
50 |     Linear<float, 50, 10> fc2;
51 | 
52 |     void learn(float lr)
53 |     {
54 |       c1.learn(lr);
55 |       c2.learn(lr);
56 |       fc1.learn(lr);
57 |       fc2.learn(lr);
58 |     }
59 | 
60 |     void save(std::ostream& out) const
61 |     {
62 |       c1.save(out);      c2.save(out);       fc1.save(out);       fc2.save(out);
63 |     }
64 |     void load(std::istream& in)
65 |     {
66 |       c1.load(in);      c2.load(in);       fc1.load(in);       fc2.load(in);
67 |     }
68 |   };
69 |   
70 |   void init(State& s)
71 |   {
72 |     img.zero();
73 |                       
74 |     auto step1 = s.c1.forward(img);
75 |     
76 |     std::array<NNArray<float, 12,12>, 10> step2;
77 |     unsigned ctr=0;
78 |     for(auto& p : step2)
79 |       p = step1[ctr++].Max2d<2>().applyFunc(ReluFunc());
80 | 
81 |     std::array<NNArray<float, 8,8>, 20> step3 = s.c2.forward(step2);
82 |     std::array<NNArray<float, 4,4>, 20> step4;
83 | 
84 |     ctr=0;
85 |     for(auto& p : step4) {
86 |       p = step3[ctr++].Max2d<2>().applyFunc(ReluFunc());
87 |     }
88 | 
89 |     NNArray<float, 4*4*20, 1> flat = flatten(step4);
90 |     auto output = s.fc1.forward(flat);
91 |     auto output2 = output.applyFunc(ReluFunc());
92 |     auto output3 = s.fc2.forward(output2);
93 | 
94 |     scores = output3.logSoftMax();
95 |     expected.zero();
96 |     loss = TrackedFloat(0) - (expected*scores)(0,0);
97 |   }
98 | };
99 | 


--------------------------------------------------------------------------------
/37learn.cc:
--------------------------------------------------------------------------------
  1 | #include "mnistreader.hh"
  2 | #include "vizi.hh"
  3 | #include <iostream>
  4 | #include "ext/sqlitewriter/sqlwriter.hh"
  5 | #include <unistd.h>
  6 | 
  7 | using namespace std;
  8 | 
  9 | float doTest(const MNISTReader& mntest, const Tensor<float>& weights, float bias, SQLiteWriter* sqw=0)
 10 | {
 11 |   unsigned int corrects=0, wrongs=0;
 12 |   
 13 |   for(unsigned int n = 0 ; n < mntest.num(); ++n) {
 14 |     int label = mntest.getLabel(n);
 15 |     if(label != 3 && label != 7)
 16 |       continue;
 17 |     Tensor img(28,28);
 18 |     mntest.pushImage(n, img);
 19 | 
 20 |     float score = (img.dot(weights).sum()(0,0)) + bias; // the calculation
 21 | 
 22 |     int predict = score > 0 ? 7 : 3;                  // the verdict
 23 | 
 24 |     if(sqw)
 25 |       sqw->addValue({{"label", label}, {"res", score}, {"verdict", predict}});
 26 | 
 27 |     
 28 |     if(predict == label) {
 29 |       corrects++;
 30 |     }
 31 |     else {
 32 |       wrongs++;
 33 |     }
 34 |   }
 35 |   float perc = 100.0*corrects/(corrects+wrongs); 
 36 |   cout << perc << "% correct" << endl;
 37 |   return perc;
 38 | }
 39 | 
 40 | int main()
 41 | {
 42 |   MNISTReader mn("gzip/emnist-digits-train-images-idx3-ubyte.gz", "gzip/emnist-digits-train-labels-idx1-ubyte.gz");
 43 |   MNISTReader mntest("gzip/emnist-digits-test-images-idx3-ubyte.gz", "gzip/emnist-digits-test-labels-idx1-ubyte.gz");
 44 | 
 45 |   cout << "Have "<<mn.num() << " training images and " << mntest.num() << " validation images." <<endl;
 46 | 
 47 |   Tensor weights(28,28);
 48 |   weights.randomize(1.0/sqrt(28*28));
 49 | 
 50 |   saveTensor(weights, "random-weights.png", 252);
 51 | 
 52 |   float bias=0;
 53 |   
 54 |   unlink("37learn.sqlite3");
 55 |   SQLiteWriter sqw("37learn.sqlite3");
 56 |   int count=0;
 57 | 
 58 |   Tensor lr(28,28);
 59 |   lr.identity(0.01);
 60 | 
 61 |   for(unsigned int n = 0 ; n < mn.num(); ++n) {
 62 |     int label = mn.getLabel(n);
 63 |     if(label != 3 && label != 7)
 64 |       continue;
 65 | 
 66 |     if(!(count % 4)) {
 67 |       if(doTest(mntest, weights, bias) > 98.0)
 68 |         break;
 69 |       saveTensor(weights, "weights-"+to_string(count)+".png", 252);
 70 |     }
 71 | 
 72 |     Tensor img(28,28);
 73 |     mn.pushImage(n, img);
 74 |     float res = (img.dot(weights).sum()(0,0)) + bias; // the calculation
 75 |     if(count == 25001) {
 76 |       auto prod = img.dot(weights);
 77 |       saveTensor(img, "random-image.png", 252, true);
 78 |       saveTensor(prod, "random-prod.png", 252);
 79 |       cout<<"res for first image: " << res << '\n';
 80 |     }
 81 |     int verdict = res > 0 ? 7 : 3;
 82 | 
 83 |     if(label == 7) {
 84 |       if(res < 2.0) {
 85 |         weights.raw() = weights.raw() + img.raw() * lr.raw();
 86 |         bias += 0.01;
 87 |       }
 88 |     } else {
 89 |       if(res > -2.0) {
 90 |         weights.raw() = weights.raw() - img.raw() * lr.raw();
 91 |         bias -= 0.01;
 92 |       }
 93 |     }
 94 |     
 95 | 
 96 |     ++count;
 97 |   }
 98 |   saveTensor(weights, "weights-final.png", 252);
 99 |   doTest(mntest, weights, bias, &sqw);
100 |   cout<<"Bias: "<<bias<<endl;
101 | }
102 | 


--------------------------------------------------------------------------------
/vizi.cc:
--------------------------------------------------------------------------------
 1 | #include "vizi.hh"
 2 | #define STB_IMAGE_IMPLEMENTATION
 3 | #include "ext/stb/stb_image.h"
 4 | #define STB_IMAGE_RESIZE_IMPLEMENTATION
 5 | #include "ext/stb/stb_image_resize.h"
 6 | #define STB_IMAGE_WRITE_IMPLEMENTATION
 7 | #include "ext/stb/stb_image_write.h"
 8 | #define STB_TRUETYPE_IMPLEMENTATION  // force following include to generate implementation
 9 | #include "ext/stb/stb_truetype.h"
10 | #include <vector>
11 | 
12 | using namespace std;
13 | 
14 | FontWriter::FontWriter()
15 | {
16 |   std::ifstream in("/usr/share/fonts/truetype/msttcorefonts/Times_New_Roman_Bold.ttf", std::ios::binary);
17 |   d_ttf_buffer.assign(std::istreambuf_iterator<char>(in),
18 |                       std::istreambuf_iterator<char>());
19 |   
20 |   stbtt_InitFont(&d_font, (const unsigned char*)&d_ttf_buffer[0], stbtt_GetFontOffsetForIndex((const unsigned char*)&d_ttf_buffer[0],0));
21 | }
22 | 
23 | void FontWriter::writeChar(char ch, int s, int c, int r, std::function<void(int, int, int, int, int)> f)
24 | {
25 |   int w,h,i,j;
26 |   unsigned char *bitmap = stbtt_GetCodepointBitmap(&d_font, 0,stbtt_ScaleForPixelHeight(&d_font, s), ch, &w, &h, 0,0);
27 |   c -= w/2; // center
28 |   for (j=0; j < h; ++j) {
29 |     for (i=0; i < w; ++i) {
30 |       f(c + i, r + j, 255-bitmap[j*w+i], 255-bitmap[j*w+i], 255-bitmap[j*w+i]);
31 |     }
32 |   }
33 | }
34 | 
35 | 
36 | void saveTensor(const Tensor<float>& t, const std::string& fname, int size, bool monochrome)
37 | {
38 |   vector<uint8_t> out;
39 |   out.resize(size*size*3);
40 |   struct Pixel {
41 |     uint8_t r, g, b;
42 |   };
43 |   static_assert(sizeof(Pixel)==3);
44 |   
45 |   auto pix = [&out, &size](int col, int row) -> Pixel&
46 |   {
47 |     return *(Pixel*)&out[3 *(col + row*size)];
48 |   };
49 | 
50 |   float lemin, lemax;
51 |   lemin = lemax = t(0,0);
52 |   
53 |   for(unsigned int row = 0 ; row < t.getRows(); ++row) {
54 |     for(unsigned int col = 0 ; col < t.getCols(); ++col) {
55 |       float v = t(row, col);
56 |       if(v > lemax)
57 |         lemax = v;
58 |       if(v < lemin)
59 |         lemin = v;
60 |     }
61 |   }
62 | 
63 |   unsigned int hboxsize = size/t.getCols();
64 |   unsigned int vboxsize = size/t.getRows();
65 | 
66 |   auto box = [&pix, &out](int col, int row, int w, int h, uint8_t cr, uint8_t cg, uint8_t cb) {
67 |     for(int c = col ; c < col + w; ++c)
68 |       for(int r = row ; r < row + h; ++r)
69 |         pix(c, r) = {cr,cg,cb};
70 |   };
71 |   
72 |   for(unsigned int row = 0 ; row < t.getRows(); ++row) {
73 |     for(unsigned int col = 0 ; col < t.getCols(); ++col) {
74 |       float v = t(row, col);
75 | 
76 |       if(monochrome) {
77 |         uint8_t color = 255.0*(v - lemin)/(lemax-lemin);
78 |         box(col * hboxsize, row * vboxsize, hboxsize, vboxsize, color, color, color);
79 |       }
80 |       else {
81 |         if(v > 0)  // red
82 |           box(col * hboxsize, row * vboxsize, hboxsize, vboxsize, 255 * v/lemax, 0, 0);
83 |         else if(v < 0) // blue
84 |           box(col * hboxsize, row * vboxsize, hboxsize, vboxsize, 0, 0, 255 * v/lemin);
85 |       }
86 |     }
87 |   }
88 |       
89 |   
90 |   stbi_write_png(fname.c_str(), size, size, 3, &out[0], 3*size);
91 | }
92 | 


--------------------------------------------------------------------------------
/gru-layer.hh:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include "tensor-layers.hh"
 3 | // hidden state=> dense linear => output x
 4 | 
 5 | // x is input
 6 | // h_t, h_{t-1} = hidden state
 7 | 
 8 | // gate_{reset} = \sigma(W_{input_{reset}} \cdot x_t + W_{hidden_{reset}} \cdot h_{t-1})
 9 | 
10 | // W_input_reset  - ^^ normal matrix products
11 | // W_input_hidden
12 | 
13 | // pytorch:
14 | 
15 | // r_t​ = σ(W_{ir} ​x_t + b_{ir}​+W_{hr}​h_{t−1}​ +b_{hr}​)            // reset gate
16 | // z_t​ = σ(W_{iz} ​x_t ​+ b_{iz} ​+W_{hz}​h_{t−1}​ + b_{hz}​)          // update
17 | // n_t​ = tanh(W_{in}​x_t​+b_{in}​+ r_t​*(W_{hn} ​h_{t−1}​ + b_{hn}​))   // "new" - * is dotproduct
18 | // h_t​=(1−z_t​)*n_t​+z_t​*h_{t−1)​}                                  // new h
19 | 
20 | // the hidden state is also the output, which needs linear combination to turn into input size again
21 | // https://pytorch.org/docs/stable/generated/torch.nn.GRU.html
22 | 
23 | template<typename T, unsigned int IN, unsigned int HIDDEN>
24 | struct GRULayer : TensorLayer<T>
25 | {
26 |   Tensor<T> d_w_ir{HIDDEN, IN}; // reset
27 |   Tensor<T> d_w_iz{HIDDEN, IN}; // update
28 |   Tensor<T> d_w_in{HIDDEN, IN}; // new
29 | 
30 |   Tensor<T> d_w_hr{HIDDEN, HIDDEN};  // hidden reset
31 |   Tensor<T> d_w_hz{HIDDEN, HIDDEN}; // hidden update
32 |   Tensor<T> d_w_hn{HIDDEN, HIDDEN}; // hidden "new"
33 | 
34 |   Tensor<T> d_origprevh{HIDDEN, 1};
35 |   Tensor<T> d_prevh{HIDDEN, 1};
36 | 
37 |   GRULayer()
38 |   {
39 |     this->d_params={
40 |         {&d_w_ir, "w_ir"},         {&d_w_iz, "w_iz"},         {&d_w_in, "w_in"},
41 |         {&d_w_hr, "w_hr"},         {&d_w_hz, "w_hz"},         {&d_w_hn, "w_hn"}};
42 |     randomize();
43 |     Tensor one(HIDDEN, HIDDEN);
44 |     one.identity(1.0);
45 |     d_prevh = one*d_origprevh;
46 |   }
47 | 
48 |   // https://blog.floydhub.com/gru-with-pytorch/
49 |   //  https://towardsdatascience.com/gate-recurrent-units-explained-using-matrices-part-1-3c781469fc18
50 |   // these appear to be slightly different
51 |   auto forward(const Tensor<T>& xt)
52 |   {
53 |     auto r_t = makeFunction<SigmoidFunc>(d_w_ir * xt + d_w_hr * d_prevh); // reset gate
54 |     auto z_t = makeFunction<SigmoidFunc>(d_w_iz * xt + d_w_hz * d_prevh);
55 |     // z_t dimensions: rows from d_w_iz, columns from xt -> HIDDEN,IN
56 |     auto n_t = makeFunction<TanhFunc>(d_w_in * xt + r_t.dot(d_w_hn *d_prevh));
57 | 
58 |     Tensor<T> one(HIDDEN, 1);  // XXX this is a SUPER wart
59 |     // the problem is we have no support for 1 - Tensor() kind of operations, so we need to make an appropriately sized 'one'
60 |     for(unsigned int r=0 ; r < one.getRows(); ++r)
61 |       for(unsigned int c=0 ; c < one.getCols(); ++c)
62 |         one(r,c)=1;
63 |     
64 |     auto h_t = (one - z_t).dot(n_t) + z_t.dot(d_prevh);
65 |     d_prevh = h_t; // "this is where the magic happens"
66 |     return h_t;
67 |   }
68 | 
69 |   void randomize() // "Xavier initialization"  http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf
70 |   {
71 |     d_w_ir.randomize(1.0/sqrt(HIDDEN));
72 |     d_w_iz.randomize(1.0/sqrt(HIDDEN));
73 |     d_w_in.randomize(1.0/sqrt(HIDDEN));
74 | 
75 |     d_w_hr.randomize(1.0/sqrt(HIDDEN));
76 |     d_w_hz.randomize(1.0/sqrt(HIDDEN));
77 |     d_w_hn.randomize(1.0/sqrt(HIDDEN));
78 |     d_prevh.zero();
79 |   }
80 | };
81 | 


--------------------------------------------------------------------------------
/ext/sqlitewriter/sqlwriter.hh:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | #include <string>
 3 | #include <vector>
 4 | #include <unordered_map>
 5 | #include <variant>
 6 | #include <mutex>
 7 | #include <thread>
 8 | #include <iostream>
 9 | 
10 | struct sqlite3;
11 | struct sqlite3_stmt;
12 | 
13 | class MiniSQLite
14 | {
15 | public:
16 |   MiniSQLite(std::string_view fname);
17 |   ~MiniSQLite();
18 |   std::vector<std::pair<std::string, std::string>> getSchema(const std::string& table);
19 |   void addColumn(const std::string& table, std::string_view name, std::string_view type);
20 |   std::vector<std::vector<std::string>> exec(std::string_view query);
21 |   void prepare(const std::string& table, std::string_view str);
22 |   void bindPrep(const std::string& table, int idx, bool value);
23 |   void bindPrep(const std::string& table, int idx, int value);
24 |   void bindPrep(const std::string& table, int idx, uint32_t value);
25 |   void bindPrep(const std::string& table, int idx, long value);
26 |   void bindPrep(const std::string& table, int idx, unsigned long value);
27 |   void bindPrep(const std::string& table, int idx, long long value); 
28 |   void bindPrep(const std::string& table, int idx, unsigned long long value);
29 |   void bindPrep(const std::string& table, int idx, double value);
30 |   void bindPrep(const std::string& table, int idx, const std::string& value);
31 |   void execPrep(const std::string& table); 
32 |   void begin();
33 |   void commit();
34 |   void cycle();
35 |   bool isPrepared(const std::string& table) const
36 |   {
37 |     if(auto iter = d_stmts.find(table); iter == d_stmts.end())
38 |       return false;
39 |     else
40 |       return iter->second != nullptr;
41 |   }
42 | 
43 | private:
44 |   sqlite3* d_sqlite;
45 |   std::unordered_map<std::string, sqlite3_stmt*> d_stmts;
46 |   std::vector<std::vector<std::string>> d_rows; // for exec()
47 |   static int helperFunc(void* ptr, int cols, char** colvals, char** colnames);
48 |   bool d_intransaction{false};
49 |   bool haveTable(const std::string& table);
50 | };
51 | 
52 | class SQLiteWriter
53 | {
54 | 
55 | public:
56 |   explicit SQLiteWriter(std::string_view fname) : d_db(fname)
57 |   {
58 |     //    for(const auto& c : d_columns)
59 |     //      cout <<c.first<<"\t"<<c.second<<endl;
60 | 
61 |     d_db.exec("PRAGMA journal_mode='wal'");
62 |     d_db.begin(); // open the transaction
63 |     d_thread = std::thread(&SQLiteWriter::commitThread, this);
64 |   }
65 |   typedef std::variant<double, int32_t, uint32_t, int64_t, std::string> var_t;
66 |   void addValue(const std::initializer_list<std::pair<const char*, var_t>>& values, const std::string& table="data");
67 |   void addValue(const std::vector<std::pair<const char*, var_t>>& values, const std::string& table="data");
68 |   
69 |   template<typename T>
70 |   void addValueGeneric(const std::string& table, const T& values);
71 |   ~SQLiteWriter()
72 |   {
73 |     //    std::cerr<<"Destructor called"<<std::endl;
74 |     d_pleasequit=true;
75 |     d_thread.join();
76 |   }
77 | 
78 | private:
79 |   void commitThread();
80 |   bool d_pleasequit{false};
81 |   std::thread d_thread;
82 |   std::mutex d_mutex;  
83 |   MiniSQLite d_db;
84 |   std::unordered_map<std::string, std::vector<std::pair<std::string, std::string>>> d_columns;
85 |   std::unordered_map<std::string, std::vector<std::string>> d_lastsig;
86 |   bool haveColumn(const std::string& table, std::string_view name);
87 | 
88 | };
89 | 


--------------------------------------------------------------------------------
/threeorseven.cc:
--------------------------------------------------------------------------------
  1 | #include "mnistreader.hh"
  2 | #include "vizi.hh"
  3 | #include <iostream>
  4 | #include "ext/sqlitewriter/sqlwriter.hh"
  5 | #include <unistd.h>
  6 | 
  7 | using namespace std;
  8 | 
  9 | int main()
 10 | {
 11 |   MNISTReader mn("gzip/emnist-digits-train-images-idx3-ubyte.gz", "gzip/emnist-digits-train-labels-idx1-ubyte.gz");
 12 |   MNISTReader mntest("gzip/emnist-digits-test-images-idx3-ubyte.gz", "gzip/emnist-digits-test-labels-idx1-ubyte.gz");
 13 | 
 14 |   cout << "Have "<<mn.num() << " training images and " << mntest.num() << " validation images." <<endl;
 15 | 
 16 |   Tensor threes(28, 28), sevens(28, 28);
 17 |   
 18 |   float threecount = 0, sevencount=0;
 19 |   for(unsigned int n = 0 ; n < mn.num(); ++n) {
 20 |     int label = mn.getLabel(n);
 21 |     if(label != 3 && label != 7)
 22 |       continue;
 23 |     
 24 |     Tensor img(28,28);
 25 |     mn.pushImage(n, img);
 26 | 
 27 |     if(label == 3) {
 28 |       threecount++;
 29 |       threes.raw() += img.raw();
 30 |     }
 31 |     else {
 32 |       sevencount++;
 33 |       sevens.raw() += img.raw();
 34 |     }
 35 |   }
 36 |   saveTensor(threes, "threes.png", 252, true);
 37 |   saveTensor(sevens, "sevens.png", 252, true);
 38 | 
 39 |   Tensor totcount(threecount + sevencount);
 40 |   auto delta = (sevens - threes) / totcount;
 41 | 
 42 |   saveTensor(delta, "diff.png", 252);
 43 | 
 44 |   float threesmean = 0, sevensmean = 0;
 45 | 
 46 |   unlink("threeorseven.sqlite3");
 47 |   SQLiteWriter sqw("threeorseven.sqlite3");
 48 |   
 49 |   for(unsigned int n = 0 ; n < mn.num(); ++n) {
 50 |     int label = mn.getLabel(n);
 51 |     if(label != 3 && label != 7)
 52 |       continue;
 53 |     Tensor img(28,28);
 54 |     mn.pushImage(n, img);
 55 | 
 56 |     float res = (img.dot(delta).sum()(0,0)); // the calculation
 57 | 
 58 |     sqw.addValue({{"label", label}, {"res", res}});
 59 |     if(label == 3) 
 60 |       threesmean += res;
 61 |     else 
 62 |       sevensmean += res;
 63 |   }
 64 | 
 65 |   cout<<"Three average result: "<<threesmean/threecount<<", seven average result: "<<sevensmean/sevencount<<endl;
 66 |   float middle = (sevensmean/sevencount + threesmean/threecount) / 2;
 67 |   //  middle = 0;
 68 |   cout<<"Middle: "<< middle <<endl;
 69 | 
 70 |   float bias = -middle;
 71 |   
 72 |   unsigned int corrects=0, wrongs=0;
 73 |   int haveseven=0, havethree=0;
 74 |   for(unsigned int n = 0 ; n < mntest.num(); ++n) {
 75 |     int label = mntest.getLabel(n);
 76 |     if(label != 3 && label != 7)
 77 |       continue;
 78 |     Tensor img(28,28);
 79 |     mntest.pushImage(n, img);
 80 | 
 81 |     float score = (img.dot(delta).sum()(0,0)) + bias; // the calculation
 82 |     int predict = score > 0 ? 7 : 3;                  // the verdict
 83 |     
 84 |     if(predict == label) {
 85 |       if(haveseven < 5 && label==7) {
 86 |         saveTensor(img, "seven-"+to_string(haveseven)+".png", 252, true);
 87 |         Tensor prod  = img.dot(delta);
 88 |         saveTensor(prod, "prod7-"+to_string(haveseven)+".png", 252);
 89 |         haveseven++;
 90 |       }
 91 |       if(havethree < 5 && label==3) {
 92 |         saveTensor(img, "three-"+to_string(havethree)+".png", 252, true);
 93 |         Tensor prod  = img.dot(delta);
 94 |         saveTensor(prod, "prod3-"+to_string(havethree)+".png", 252);
 95 |         havethree++;
 96 |       }
 97 | 
 98 |       corrects++;
 99 |     }
100 |     else {
101 |       saveTensor(img, "wrong-"+to_string(label)+"-"+to_string(wrongs)+".png", 252);
102 |       wrongs++;
103 |     }
104 |   }
105 |   cout << 100.0*corrects/(corrects+wrongs) << "% correct" << endl;
106 | }
107 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
  1 | cmake_minimum_required(VERSION 3.1)
  2 | set(CMAKE_CXX_FLAGS "-Wall -Wextra -O3 -ggdb -march=native -msse2 -mavx -mavx2 -mfma  -fno-omit-frame-pointer -ffast-math -Iext/imgui -Iext/imgui/backends -Iext/implot-master -I/usr/include/eigen3 -I/usr/include/eigen3/unsupported/ -Wno-missing-field-initializers")
  3 | 
  4 | project(hello-dl VERSION 1.0
  5 |                   DESCRIPTION "Hello deep learning"
  6 |                   LANGUAGES CXX)
  7 | 
  8 | 
  9 | #add_compile_options(-fsanitize=address)
 10 | #add_link_options(-fsanitize=address)
 11 |                 
 12 | set(CMAKE_CXX_STANDARD 17 CACHE STRING "The C++ standard to use")
 13 | set(CMAKE_CXX_STANDARD_REQUIRED ON)
 14 | set(CMAKE_CXX_EXTENSIONS ON)
 15 | 
 16 | set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
 17 | set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 18 | find_package(Threads REQUIRED)
 19 | find_package (Eigen3 3.3 REQUIRED NO_MODULE)
 20 | 
 21 | add_library(common OBJECT mnistreader.cc textsupport.cc )
 22 | #add_library(gui OBJECT ui.cc ext/imgui/imgui.cpp  ext/imgui/imgui_demo.cpp  ext/imgui/imgui_draw.cpp  ext/imgui/imgui_tables.cpp  ext/imgui/imgui_widgets.cpp ext/imgui/backends/imgui_impl_glfw.cpp ext/imgui/backends/imgui_impl_opengl3.cpp ext/implot-master/implot.cpp ext/implot-master/implot_items.cpp ext/implot-master/implot_demo.cpp)
 23 | 
 24 | 
 25 | 
 26 | add_executable(tensor-relu tensor-relu.cc ext/sqlitewriter/sqlwriter.cc)
 27 | target_link_libraries(tensor-relu z  common Eigen3::Eigen sqlite3 Threads::Threads)
 28 | 
 29 | add_executable(tensor-convo tensor-convo.cc ext/sqlitewriter/sqlwriter.cc )
 30 | target_link_libraries(tensor-convo z  common Eigen3::Eigen sqlite3 Threads::Threads) 
 31 | 
 32 | add_executable(tensor-convo-par tensor-convo-par.cc ext/sqlitewriter/sqlwriter.cc )
 33 | target_link_libraries(tensor-convo-par z common Eigen3::Eigen sqlite3 Threads::Threads) 
 34 | 
 35 | 
 36 | add_executable(img-ocr img-ocr.cc vizi.cc )
 37 | target_link_libraries(img-ocr z  common Eigen3::Eigen Threads::Threads) 
 38 | 
 39 | add_executable(threeorseven threeorseven.cc vizi.cc ext/sqlitewriter/sqlwriter.cc)
 40 | target_link_libraries(threeorseven z  common Eigen3::Eigen sqlite3 Threads::Threads) 
 41 | 
 42 | add_executable(37learn 37learn.cc vizi.cc ext/sqlitewriter/sqlwriter.cc)
 43 | target_link_libraries(37learn z  common Eigen3::Eigen sqlite3 Threads::Threads) 
 44 | 
 45 | add_executable(basic-autograd basic-autograd.cc)
 46 | target_link_libraries(basic-autograd z  common Eigen3::Eigen Threads::Threads ) 
 47 | 
 48 | add_executable(imagine imagine.cc vizi.cc)
 49 | target_link_libraries(imagine z  common Eigen3::Eigen Threads::Threads ) 
 50 | 
 51 | 
 52 | add_executable(tensor-gru tensor-gru.cc )
 53 | target_link_libraries(tensor-gru z  common Eigen3::Eigen Threads::Threads )
 54 | 
 55 | add_executable(testrunner testrunner.cc arraytests.cc testfvector.cc grutest.cc tensor2test.cc tensormodtest.cc)
 56 | 
 57 | target_link_libraries(testrunner  z  common Eigen3::Eigen)
 58 | 
 59 | enable_testing()
 60 | add_test(testname testrunner)
 61 | 
 62 | 
 63 | 
 64 | #add_executable(first-convo first-convo.cc )
 65 | #target_link_libraries(first-convo z  common)
 66 | 
 67 | #add_executable(modular-convo modular-convo.cc )
 68 | #target_link_libraries(modular-convo z  common)
 69 | 
 70 | #add_executable(modular-linear-convo modular-linear-convo.cc )
 71 | #target_link_libraries(modular-linear-convo z  common)
 72 | 
 73 | 
 74 | #add_executable(worker-convo worker-convo.cc  )
 75 | #target_link_libraries(worker-convo  z  common)
 76 | 
 77 | #add_executable(worker-convo-avx worker-convo-avx.cc)
 78 | #target_link_libraries(worker-convo-avx  z  common gui GL glfw)
 79 | 
 80 | #add_executable(worker-convo-avx-threaded worker-convo-avx-threaded.cc )
 81 | #target_link_libraries(worker-convo-avx-threaded z  common)
 82 | 
 83 | 
 84 | #add_executable(modular-threaded-convo modular-threaded-convo.cc )
 85 | #target_link_libraries(modular-threaded-convo z  common)
 86 | 
 87 | 
 88 | add_executable(try-convo try-convo.cc vizi.cc)
 89 | target_link_libraries(try-convo z  common Threads::Threads)
 90 | 
 91 | 
 92 | #add_executable(los los.cc )
 93 | #target_link_libraries(los z  common)
 94 | 
 95 | add_executable(mnistposter mnistposter.cc )
 96 | target_link_libraries(mnistposter z  common)
 97 | 
 98 | #add_executable(los2 los2.cc )
 99 | #target_link_libraries(los2 z  common)
100 | 
101 | #add_executable(los3 los3.cc )
102 | #target_link_libraries(los3 z  common)
103 | 
104 | #add_executable(tensor tensor.cc)
105 | #target_link_libraries(tensor z  common)
106 | 
107 | #add_executable(first-relu first-relu.cc )
108 | #target_link_libraries(first-relu z  common)
109 | 
110 | 
111 | 


--------------------------------------------------------------------------------
/cnn-alphabet.hh:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include "model.hh"
  3 | #include <sstream>
  4 | /*
  5 | Model from https://data-flair.training/blogs/handwritten-character-recognition-neural-network/
  6 |  */
  7 | 
  8 | 
  9 | template<typename T>
 10 | struct CNNAlphabetModel  {
 11 |   NNArray<T, 28, 28> img;
 12 | 
 13 |   NNArray<T, 26, 1> scores;
 14 |   NNArray<T, 1, 26> expected;
 15 | 
 16 |   TrackedNumber<T> loss;
 17 | 
 18 |   struct State : ModelState
 19 |   {
 20 |            //      R   C  K IN  OUTLAYERS
 21 |     Conv2d<T, 28, 28, 3, 1,  32> c1; // -> 26*26 -> max2d -> 13*13
 22 |     Conv2d<T, 13, 13, 3, 32, 64> c2; // -> -> 11*11 -> max2d -> 6*6 //padding
 23 |     Conv2d<T, 6,   6, 3, 64, 128> c3; // -> 4*4 -> max2d -> 2*2
 24 |     // flattened to 512
 25 |            //      IN OUT
 26 |     Linear<T, 512, 64> fc1;  
 27 |     Linear<T, 64, 128> fc2;
 28 |     Linear<T, 128, 26> fc3;
 29 | 
 30 |         // this has GRAVE issues with copying
 31 |     State() 
 32 |     {    // this has GRAVE issues with copying
 33 |       d_members={&c1, &c2, &c3, &fc1, &fc2, &fc3};
 34 |     }
 35 |     // this has GRAVE issues with copying
 36 |     State(const State& rhs) 
 37 |     {
 38 |       c1=rhs.c1;
 39 |       c2=rhs.c2;
 40 |       c3=rhs.c3;
 41 |       fc1=rhs.fc1;
 42 |       fc2=rhs.fc2;
 43 |       fc3=rhs.fc3;
 44 |       d_members={&c1, &c2, &c3, &fc1, &fc2, &fc3};
 45 |     }
 46 | 
 47 |     State& operator=(const State& rhs) = delete;
 48 |     
 49 |     void reset()
 50 |     {
 51 |       for(auto& m : d_members)
 52 |         m->reset();
 53 |     }
 54 |     
 55 |     void addGrad(const State& rhs)
 56 |     {
 57 |       c1.addGrad(rhs.c1);
 58 |       c2.addGrad(rhs.c2);
 59 |       c3.addGrad(rhs.c3);
 60 | 
 61 |       fc1.addGrad(rhs.fc1);
 62 |       fc2.addGrad(rhs.fc2);
 63 |       fc3.addGrad(rhs.fc3);
 64 |     }
 65 | 
 66 |     void setGrad(const State& rhs, float divisor)
 67 |     {
 68 |       c1.setGrad(rhs.c1, divisor);
 69 |       c2.setGrad(rhs.c2, divisor);
 70 |       c3.setGrad(rhs.c3, divisor);
 71 | 
 72 |       fc1.setGrad(rhs.fc1, divisor);
 73 |       fc2.setGrad(rhs.fc2, divisor);
 74 |       fc3.setGrad(rhs.fc3, divisor);
 75 |     }
 76 | 
 77 |     void momentum(const State& rhs, float mom, float dampening=0)
 78 |     {
 79 |       c1.momGrad(rhs.c1, mom, dampening);
 80 |       c2.momGrad(rhs.c2, mom, dampening);
 81 |       c3.momGrad(rhs.c3, mom, dampening);
 82 | 
 83 |       fc1.momGrad(rhs.fc1, mom, dampening);
 84 |       fc2.momGrad(rhs.fc2, mom, dampening);
 85 |       fc3.momGrad(rhs.fc3, mom, dampening);
 86 |     }
 87 | 
 88 |     
 89 |     template<typename W>
 90 |     void makeProj(const W& w)
 91 |     {
 92 |       fc1.makeProj(w);
 93 |       fc2.makeProj(w);
 94 |       fc3.makeProj(w);
 95 |       c1.makeProj(w);
 96 |       c2.makeProj(w);
 97 |     }
 98 | 
 99 |     template<typename W>
100 |     void projForward(W& w) const
101 |     {
102 |       fc1.projForward(w);
103 |       fc2.projForward(w);
104 |       fc3.projForward(w);
105 |       c1.projForward(w);
106 |       c2.projForward(w);
107 |     }
108 |     template<typename W>
109 |     void projBackGrad(const W& w) 
110 |     {
111 |       fc1.projBackGrad(w);
112 |       fc2.projBackGrad(w);
113 |       fc3.projBackGrad(w);
114 |       c1.projBackGrad(w);
115 |       c2.projBackGrad(w);
116 |     }
117 | 
118 |   };
119 |   
120 |   void init(State& s)
121 |   {
122 |     img.zero();
123 |     img.setVariable();
124 |                       
125 |     auto step1 = s.c1.forward(img);
126 |     
127 |     std::array<NNArray<T, 13,13>, 32> step2;
128 |     unsigned ctr=0;
129 |     for(auto& p : step2)
130 |       p = step1[ctr++]. template Max2d<2>().applyFunc(ReluFunc());
131 | 
132 |     std::array<NNArray<T, 11,11>, 64> step3 = s.c2.forward(step2);
133 |     std::array<NNArray<T, 6,6>, 64> step4;
134 | 
135 |     ctr=0;
136 |     for(auto& p : step4) {
137 |       p = step3[ctr++]. template Max2d<2>().applyFunc(ReluFunc());
138 |     }
139 | 
140 |     std::array<NNArray<T, 4,4>, 128> step5 = s.c3.forward(step4);
141 |     std::array<NNArray<T, 2,2>, 128> step6;
142 | 
143 |     ctr=0;
144 |     for(auto& p : step6) {
145 |       p = step5[ctr++]. template Max2d<2>().applyFunc(ReluFunc());
146 |     }
147 |     
148 |     NNArray<T, 2*2*128, 1> flat = flatten(step6);
149 |     auto output = s.fc1.forward(flat);
150 |     auto output2 = output.applyFunc(ReluFunc());
151 |     auto output3 = s.fc2.forward(output2).applyFunc(ReluFunc());
152 |     auto output4 = s.fc3.forward(output3).applyFunc(ReluFunc());
153 | 
154 |     scores = output4.logSoftMax();
155 |     expected.zero();
156 |     loss = TrackedNumber<T>(0.0) - (expected*scores)(0,0);
157 |   }
158 | 
159 | };
160 | 


--------------------------------------------------------------------------------
/testfvector.cc:
--------------------------------------------------------------------------------
  1 | #include "ext/doctest.h"
  2 | #include "array.hh"
  3 | #include <sstream>
  4 | #include "fvector.hh"
  5 | #include "tracked.hh"
  6 | using namespace std;
  7 | 
  8 | TEST_CASE("basic fvect4 test") {
  9 |   fvector<4> val=1234;
 10 |   CHECK(val.v[0] == 1234);
 11 |   CHECK(val.v[1] == 1234);
 12 |   CHECK(val.v[2] == 1234);
 13 |   CHECK(val.v[3] == 1234);
 14 | 
 15 |   CHECK(val.sum() == 4*1234);
 16 | 
 17 |   val.v[0]++;
 18 |   val *= 2;
 19 | 
 20 |   CHECK(val.v[0] == 2*1235);
 21 |   
 22 | }
 23 | 
 24 | 
 25 | TEST_CASE("basic fvect8 test") {
 26 |   fvector<8> val=1234;
 27 |   CHECK(val.v[0] == 1234);
 28 |   CHECK(val.v[1] == 1234);
 29 |   CHECK(val.v[2] == 1234);
 30 |   CHECK(val.v[3] == 1234);
 31 | 
 32 |   CHECK(val.sum() == 8*1234);
 33 | 
 34 |   val.v[0]++;
 35 |   val *= 2;
 36 | 
 37 |   CHECK(val.v[0] == 2*1235);
 38 |   
 39 | }
 40 | 
 41 | TEST_CASE("dev fvect8 test") {
 42 |   fvector<8> val({1.0, 2.0, 3.0, 4., 5., 6., 7., 8.});
 43 |   auto inv = 1.0/val;
 44 | 
 45 |   CHECK(inv.v[0]==doctest::Approx(1.0));
 46 |   CHECK(inv.v[4]==doctest::Approx(0.2));
 47 | }
 48 | 
 49 | TEST_CASE("fvect8 cmp test") {
 50 |   fvector<8> x({1,2,3,4,5,6,7,8});
 51 |   fvector<8> y({5,4,3,2,1,0,0,0});
 52 |   fvector<8> r = x<y;
 53 |   CHECK(r.v[0] == 1);
 54 |   CHECK(r.v[1] == 1);
 55 |   CHECK(r.v[2] == 0);
 56 |   CHECK(r.v[3] == 0);
 57 |   CHECK(r.v[4] == 0);
 58 |   CHECK(r.v[5] == 0);
 59 |   CHECK(r.v[6] == 0);
 60 |   CHECK(r.v[7] == 0);
 61 |   
 62 |   
 63 | }
 64 | 
 65 | TEST_CASE("fvect8 max test") {
 66 |   fvector<8> x({1,2,3,4,5,6,7,8});
 67 |   fvector<8> y({5,4,3,2,1,0,0,0});
 68 |   fvector<8> sum1=0, sum2=0;
 69 |   fvector<8> res = x<y;
 70 | 
 71 | 
 72 |   
 73 |   sum1 += res * x;
 74 |   sum2 += (!res) * x;
 75 | 
 76 |   //  cout<<"sum1: "<<sum1<<endl;
 77 |   //  cout<<"sum2: "<<sum2<<endl;
 78 |   //  cout<<"comp: "<<(sum1+sum2)<<endl;
 79 |   CHECK((sum1+sum2) == x);
 80 |   auto res2= maxFunc(x,y);
 81 |   CHECK(res2.v[0] == 5);
 82 |   CHECK(res2.v[1] == 4);
 83 |   
 84 | }
 85 | 
 86 | typedef TrackedNumber<fvector<8>> TrackedVec;
 87 | TEST_CASE("tracked fvect8 test") {
 88 |   /*
 89 |   TrackedVec a = fvector<8>({-3,-2,-1,0,1,2,3,4});
 90 |   TrackedVec b = makeFunc(a, ReluFunc()) + a;
 91 |   
 92 |   fvector<8> res = b.getVal();
 93 |   cout<<"res: "<<res<<endl;
 94 |   b.backward();
 95 |   cout<<a.getGrad()<<endl;
 96 |   */
 97 | }
 98 | 
 99 | TEST_CASE("tracked max test") {
100 |   TrackedVec a = fvector<8>({-3,-2,-1, 0,1,2,3,4});
101 |   cout<<a.getVal()<<endl;
102 |   fvector<8> tmp({ 2, -9,-4, 3,2,1,4,7});
103 |   TrackedVec b = fvector<8>(tmp);
104 | 
105 |   //    -3,-2,-1,0,1,2,3,4
106 |   //b:   2 -9 -4 3 2 1 4 7 
107 |   //res: 2 -2 -1 3 2 2 4 7 
108 | 
109 | 
110 |   //  cout<< "a: "<<a.getVal()<<endl;
111 |   //  cout<< "b: "<<b.getVal()<<endl;
112 | 
113 |   
114 |   auto res = makeMax(a,b);
115 |   //  cout<<"m: "<<res.getVal()<<endl;
116 |   res.backward();
117 |   cout<<a.getGrad()<<endl;
118 |   cout<<b.getGrad()<<endl;
119 |   //  0 1 1 0 0 1 0 0 
120 |   //  1 0 0 1 1 0 1 1
121 | 
122 |   CHECK(a.getGrad().v[0] == 0);
123 |   CHECK(a.getGrad().v[1] == 1);
124 |   CHECK(a.getGrad().v[2] == 1);
125 |   CHECK(a.getGrad().v[3] == 0);
126 |   CHECK(a.getGrad().v[4] == 0);
127 |   CHECK(a.getGrad().v[5] == 1);
128 |   CHECK(a.getGrad().v[6] == 0);
129 |   CHECK(a.getGrad().v[7] == 0);
130 | 
131 |   CHECK(b.getGrad().v[0] == 1);
132 |   CHECK(b.getGrad().v[1] == 0);
133 |   CHECK(b.getGrad().v[2] == 0);
134 |   CHECK(b.getGrad().v[3] == 1);
135 |   CHECK(b.getGrad().v[4] == 1);
136 |   CHECK(b.getGrad().v[5] == 0);
137 |   CHECK(b.getGrad().v[6] == 1);
138 |   CHECK(b.getGrad().v[7] == 1);
139 | 
140 |   
141 | }
142 | 
143 | TEST_CASE("tracked log test") {
144 |   TrackedVec a = fvector<8>({3,2,1, 1,1,2,3,4});
145 |   auto res = makeFunc(a, LogFunc());
146 |   cout<<"m: "<<res.getVal()<<endl;
147 |   res.backward();
148 |   cout<<a.getGrad()<<endl;
149 |   
150 | }
151 | 
152 | TEST_CASE("assignment test") {
153 |   fvector<8> a({1,2,3});
154 |   CHECK(a.v[0]==1);
155 |   CHECK(a.v[2]==3);
156 |   CHECK(a.v[3]==0);
157 |   a=1.0;
158 |   a=fvector<8>({0});
159 |   CHECK(a.v[5]==0.0);
160 | }
161 | 
162 | 
163 | TEST_CASE("tanh test") {
164 |   TrackedVec x(1.0);
165 |   TrackedVec y(1.0);
166 |   TrackedVec res = makeFunc(x-y, TanhFunc());
167 |   
168 |   CHECK(res.getVal().v[5] == doctest::Approx(0.0));
169 |   res.backward();
170 |   CHECK(x.getGrad().v[4] == doctest::Approx(1.0));
171 |   CHECK(y.getGrad().v[3] == doctest::Approx(-1.0));
172 | 
173 |   res.zeroGrad();
174 |   x = 2;
175 |   CHECK(res.getVal().v[7] ==   doctest::Approx(tanhf(1.0)));
176 |   res.backward();
177 |   CHECK(x.getGrad().v[0] == doctest::Approx(1 - tanhf(1.0)*tanhf(1.0)));
178 |   CHECK(y.getGrad().v[1] == -x.getGrad().v[6]);
179 |   
180 | }
181 | 
182 |   
183 |   
184 | 


--------------------------------------------------------------------------------
/fvector.hh:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include <array>
  3 | #include <numeric>
  4 | #include <iostream>
  5 | 
  6 | template<unsigned int W>
  7 | struct fvector 
  8 | {
  9 |   float v __attribute__((vector_size (4*W)));
 10 |   fvector()
 11 |   {}
 12 |   fvector(float v0)
 13 |   {
 14 |     for(unsigned int n=0 ; n < W; ++n)
 15 |       v[n] = v0; 
 16 |   }
 17 | 
 18 |   // be VERY careful, fvector<8>{0.0} will call this
 19 |   fvector(const std::initializer_list<float>& in)
 20 |   {
 21 |     unsigned char ctr=0;
 22 |     for(const auto& val: in)
 23 |       v[ctr++] = val;
 24 |     for(; ctr < W; ++ctr)
 25 |       v[ctr] = 0.0;
 26 |   }
 27 | 
 28 |   fvector& operator=(float val)
 29 |   {
 30 |     *this = fvector(val);
 31 |     return *this;
 32 |   }
 33 | 
 34 |   fvector operator+(const fvector& rhs)
 35 |   {
 36 |     fvector ret = *this;
 37 |     ret.v += rhs.v;
 38 |     return ret;
 39 |   }
 40 |   fvector operator-(const fvector& rhs) const
 41 |   {
 42 |     fvector ret = *this;
 43 |     ret.v -= rhs.v;
 44 |     return ret;
 45 |   }
 46 |   fvector operator-() const
 47 |   {
 48 |     fvector ret = *this;
 49 |     ret.v = -ret.v;
 50 |     return ret;
 51 |   }
 52 | 
 53 |   fvector operator!() const
 54 |   {
 55 |     fvector ret = *this;
 56 |     for(unsigned int n = 0 ; n < W; ++n)
 57 |       ret.v[n] = !v[n];
 58 | 
 59 |     return ret;
 60 |   }
 61 | 
 62 |   fvector operator*(const float& rhs) const
 63 |   {
 64 |     fvector ret = *this;
 65 |     ret.v *= rhs;
 66 |     return ret;
 67 |   }
 68 |   
 69 |   fvector operator*(const fvector& rhs) const
 70 |   {
 71 |     fvector ret = *this;
 72 |     ret.v *= rhs.v;
 73 |     return ret;
 74 |   }
 75 |   fvector operator/(const fvector& rhs) const
 76 |   {
 77 |     fvector ret = *this;
 78 |     ret.v /= rhs.v;
 79 |     return ret;
 80 |   }
 81 |   fvector& operator/=(const fvector& rhs)
 82 |   {
 83 |     v /= rhs.v;
 84 |     return *this;
 85 |   }
 86 |   fvector& operator*=(float rhs)
 87 |   {
 88 |     v *= rhs;
 89 |     return *this;
 90 |   }
 91 | 
 92 |   fvector& operator*=(const fvector& rhs)
 93 |   {
 94 |     v *= rhs.v;
 95 |     return *this;
 96 |   }
 97 |   fvector& operator+=(const fvector& rhs)
 98 |   {
 99 |     v += rhs.v;
100 |     return *this;
101 |   }
102 |   fvector& operator+=(float rhs)
103 |   {
104 |     v += rhs;
105 |     return *this;
106 |   }
107 |   
108 |   fvector operator<(const fvector& rhs) const
109 |   {
110 |     fvector ret;
111 |     // you'd think you'd be able to do this:
112 |     //    ret.v = (v < rhs.v);
113 |     // but the output is Weird - if the comparison is valid, the output is 0 (!)
114 |     // and if invalid, it is NaN (!!) https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html
115 | 
116 |     for(unsigned int i = 0 ; i < W; ++i)
117 |       ret.v[i] = v[i] < rhs.v[i];
118 | 
119 |     return ret;
120 |   }
121 | 
122 |   bool operator==(const fvector& rhs) const
123 |   {
124 |     for(unsigned int n=0; n < W; ++n)
125 |       if(v[n] != rhs.v[n])
126 |         return false;
127 |     return true;
128 |   }
129 | 
130 |   float sum() const
131 |   {
132 |     float ret = 0;
133 |     for(unsigned int n=0; n < W; ++n)
134 |       ret += v[n];
135 |     return ret;
136 |   }
137 | };
138 | 
139 | template<unsigned int T>
140 | auto operator*(float v, const fvector<T>& rhs)
141 | {
142 |   return rhs * v;
143 | }
144 | 
145 | template<unsigned int T>
146 | auto operator/(float v, const fvector<T>& rhs)
147 | {
148 |   fvector<T> ret = v;
149 |   return ret/rhs;
150 | }
151 | 
152 | template<unsigned int T>
153 | auto operator+(float v, const fvector<T>& rhs)
154 | {
155 |   fvector<T> ret = v;
156 |   return ret+rhs;
157 | }
158 | 
159 | 
160 | template<unsigned int W>
161 | auto exp(const fvector<W>& v)
162 | {
163 |   fvector<W> ret;
164 |   for(size_t i = 0; i < W ; ++i) {
165 |     ret.v[i] = expf(v.v[i]);
166 |   }
167 |   return ret;
168 | }
169 | 
170 | template<unsigned int W>
171 | auto log(const fvector<W>& v)
172 | {
173 |   fvector<W> ret;
174 |   for(size_t i = 0; i < W ; ++i) {
175 |     ret.v[i] = logf(v.v[i]);
176 |   }
177 |   return ret;
178 | }
179 | 
180 | template<unsigned int W>
181 | auto tanh(const fvector<W>& v)
182 | {
183 |   fvector<W> ret;
184 |   for(size_t i = 0; i < W ; ++i) {
185 |     ret.v[i] = tanhf(v.v[i]);
186 |   }
187 |   return ret;
188 | }
189 | 
190 | 
191 | template<unsigned int W>
192 | std::ostream& operator<<(std::ostream& os, const fvector<W>& o) 
193 | {
194 |   for(unsigned int n=0 ; n < W; ++n)
195 |     os << o.v[n] <<" ";
196 |   return os;
197 | }
198 | /*
199 | template<unsigned int W>
200 | void setZero(fvector<W>& o)
201 | {
202 |   o=0.0;
203 | }
204 | */
205 | 
206 | template<unsigned int W>
207 | auto maxFunc(const fvector<W>& lhs, const fvector<W>& rhs)
208 | {
209 |   fvector<W> ret;
210 |   for(unsigned int i = 0 ; i < W; ++i)
211 |     ret.v[i] = std::max(lhs.v[i], rhs.v[i]);
212 |   return ret;
213 | }
214 | 
215 | 


--------------------------------------------------------------------------------
/tensor-relu.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <vector>
  3 | #include <fstream>
  4 | #include <memory>
  5 | #include <optional>
  6 | #include "tensor2.hh"
  7 | #include "mnistreader.hh"
  8 | #include "misc.hh"
  9 | #include <fenv.h>
 10 | #include "tensor-layers.hh"
 11 | #include "ext/sqlitewriter/sqlwriter.hh"
 12 | #include "vizi.hh"
 13 | using namespace std;
 14 | 
 15 | struct ReluDigitModel {
 16 |   Tensor<float> img{28,28};
 17 |   Tensor<float> scores{10, 1};
 18 |   Tensor<float> expected{1,10};
 19 |   Tensor<float> loss{1,1};
 20 |   struct State : public ModelState<float>
 21 |   { //            IN     OUT
 22 |     Linear<float, 28*28, 128> lc1;
 23 |     Linear<float, 128,    64> lc2;
 24 |     Linear<float, 64,     10> lc3;
 25 | 
 26 |     State()
 27 |     {
 28 |       this->d_members = {{&lc1, "lc1"}, {&lc2, "lc2"}, {&lc3, "lc3"}};
 29 |     }
 30 |   };
 31 |   
 32 |   void init(State& s)
 33 |   {
 34 |     auto output = s.lc1.forward(makeFlatten({img}));
 35 |     auto output2 = makeFunction<ReluFunc>(output);
 36 |     auto output3 = s.lc2.forward(output2);
 37 |     auto output4 = makeFunction<ReluFunc>(output3);
 38 |     auto output5 = s.lc3.forward(output4);
 39 |     scores = makeLogSoftMax(output5);
 40 |     loss = -(expected*scores);
 41 |   }
 42 | };
 43 | 
 44 | template<typename M>
 45 | void testModel(M& m, const MNISTReader& mn, SQLiteWriter& sqw, int batchno, bool full=false)
 46 | {
 47 |   Batcher b(mn.num());
 48 |   auto batch = b.getBatch(full ? mn.num() : 128);
 49 |   float totalLoss=0;
 50 |   int corrects=0, wrongs=0;
 51 |   auto topo = m.loss.getTopo();
 52 | 
 53 |   Tensor confusion(10, 10);
 54 |   
 55 |   for(const auto& idx : batch) {
 56 |     m.loss.zerograd(topo);
 57 |     mn.pushImage(idx, m.img);
 58 |     int label = mn.getLabel(idx);
 59 |     m.expected.zero();
 60 |     m.expected(0, label) = 1;
 61 |     
 62 |     totalLoss += m.loss(0,0); // turns it into a float
 63 |       
 64 |     int predicted = m.scores.maxValueIndexOfColumn(0);
 65 | 
 66 |     if(corrects + wrongs == 0) {
 67 |       printImgTensor(m.img);
 68 |       cout<<"predicted: "<<(int)predicted<<", actual: "<<label<<", loss: "<<m.loss<<"\n";
 69 |     }
 70 |     
 71 |     if(predicted == label)
 72 |       corrects++;
 73 |     else wrongs++;
 74 |     confusion(label, predicted)++;
 75 |   }
 76 |   double avgloss = totalLoss/batch.size();
 77 |   double corperc = (100.0*corrects/(corrects+wrongs));
 78 |   cout<<"Validation batch average loss: "<< avgloss <<", percentage correct: "<<corperc<<"%\n";
 79 |   
 80 |   sqw.addValue({{"batchno", batchno}, {"corperc", corperc}, {"avgloss", avgloss}}, "validation");
 81 |   if(full) {
 82 |     for(int r=0; r < 10; ++r)
 83 |       for(int c=0; c < 10; ++c)
 84 |         sqw.addValue({{"r", r}, {"c", c}, {"count", confusion(r,c)}}, "confusion");
 85 |   }
 86 | }
 87 | 
 88 | int main(int argc, char** argv)
 89 | {
 90 |   feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
 91 | 
 92 |   MNISTReader mn("gzip/emnist-digits-train-images-idx3-ubyte.gz", "gzip/emnist-digits-train-labels-idx1-ubyte.gz");
 93 |   MNISTReader mntest("gzip/emnist-digits-test-images-idx3-ubyte.gz", "gzip/emnist-digits-test-labels-idx1-ubyte.gz");
 94 | 
 95 |   cout<<"Have "<<mn.num()<<" training images and "<<mntest.num()<<" test images"<<endl;
 96 | 
 97 |   ReluDigitModel m;
 98 |   ReluDigitModel::State s;
 99 |   if(argc==2) {
100 |     cout<<"Loading model state from '"<<argv[1]<<"'\n";
101 |     loadModelState(s, argv[1]);
102 |   }
103 |   else
104 |     s.randomize();
105 | 
106 |   m.init(s);
107 | 
108 |   unlink("tensor-relu.sqlite3");
109 |   SQLiteWriter sqw("tensor-relu.sqlite3");
110 |   
111 |   auto topo = m.loss.getTopo();
112 |   
113 |   Batcher batcher(mn.num());
114 |   
115 |   for(unsigned int tries = 0 ;; ++tries) {
116 |     if(!(tries % 32)) {
117 |       testModel(m, mntest, sqw, tries);
118 |       saveModelState(s, "tensor-relu.state");
119 |     }
120 |     
121 |     auto batch = batcher.getBatch(64);
122 |     if(batch.empty())
123 |       break;
124 |     float totalLoss = 0;
125 |     unsigned int corrects=0, wrongs=0;
126 | 
127 |     m.loss.zeroAccumGrads(topo);
128 |     
129 |     for(const auto& idx : batch) {
130 | 
131 |       mn.pushImage(idx, m.img);
132 |       int label = mn.getLabel(idx);
133 |       m.expected.oneHotColumn(label);
134 | 
135 |       totalLoss += m.loss(0,0); // turns it into a float
136 |       
137 |       int predicted = m.scores.maxValueIndexOfColumn(0);
138 | #if 0
139 |       if(corrects + wrongs == 0) {
140 |         printImgTensor(m.img);
141 |         cout<<"predicted: "<<(int)predicted<<", actual: "<<label<<", loss: "<<m.loss<<"\n";
142 |       }
143 | #endif 
144 |       if(predicted == label)
145 |         corrects++;
146 |       else wrongs++;
147 | 
148 |       // backward the thing
149 |       m.loss.backward(topo);
150 |       m.loss.accumGrads(topo); 
151 |       // clear grads & havevalue
152 |       m.loss.zerograd(topo);
153 |     }
154 | 
155 |     double avgloss = totalLoss/batch.size();
156 |     double corperc = (100.0*corrects/(corrects+wrongs));
157 | 
158 |     cout<<tries<<": Average loss " << avgloss <<", percent batch correct "<< corperc <<"%\n";
159 | 
160 |     double lr=0.01;
161 |     sqw.addValue({{"batchno", tries}, {"corperc", corperc}, {"avgloss", avgloss}, {"lr", lr}}, "training");
162 |     
163 | 
164 |     s.learn(lr/batch.size());
165 |   }
166 |   testModel(m, mntest, sqw, mn.num()/64, true);  
167 | }
168 | 
169 | 


--------------------------------------------------------------------------------
/alternate/tensor.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <vector>
  3 | #include <fstream>
  4 | #include <memory>
  5 | #include <optional>
  6 | #include "array.hh"
  7 | #include "tracked.hh"
  8 | #include "mnistreader.hh"
  9 | #include "misc.hh"
 10 | 
 11 | using namespace std;
 12 | 
 13 | ofstream g_tree; //("tree.part");
 14 | 
 15 | struct State {
 16 |   NNArray<float, 28, 28> img;
 17 |   NNArray<float, 28*28, 1> flat;
 18 |   NNArray<float, 1, 1> output;
 19 |   int label;
 20 |   TrackedFloat score;
 21 |   TrackedFloat expected;
 22 |   TrackedFloat loss;
 23 |   void init(NNArray<float, 1, 28*28>& weights,  NNArray<float, 1, 1>& bias)
 24 |   {
 25 |     img.zero();
 26 |     flat = img.flatViewRow();
 27 |     output = weights * flat + bias;
 28 |     score = makeFunc(output(0,0), SigmoidFunc());
 29 |     expected = 0;
 30 |     loss = (expected - score) * (expected - score);
 31 |   }
 32 | };
 33 | 
 34 | 
 35 | template<typename M, typename W, typename B>
 36 | void scoreModel(W& weights, B& bias, const MNISTReader& mntest)
 37 | {
 38 |   unsigned int corrects=0, wrongs=0;
 39 |   int threes=0, sevens=0, threepreds=0, sevenpreds=0;
 40 |   M s;
 41 |   s.init(weights, bias);
 42 |   auto topo = s.score.getTopo();
 43 |   for(unsigned int i = 0 ; i < mntest.num() - 1; ++i){
 44 |     int label = mntest.getLabel(i);
 45 |     if(label==3)
 46 |       threes++;
 47 |     else if(label == 7)
 48 |       sevens++;
 49 |     else continue;
 50 | 
 51 |     mntest.pushImage(i, s.img);
 52 |     
 53 |     int verdict = s.score.getVal() < 0.5 ? 3 : 7;
 54 |     //    cout<<"label "<<(int)label<<" score "<<s.score.getVal()  <<" verdict " <<verdict<<endl;
 55 |     //cout<<"l1bias "<<l.l1.bias<<" l2bias "<<l.l2.bias<<endl;
 56 |     //    cout<<l.l1.weights.cwiseAbs().mean()<<endl;
 57 |     if(verdict == label) {
 58 |       corrects++;
 59 |     }
 60 |     else {
 61 |       wrongs++;
 62 |     }
 63 | 
 64 |     if(verdict==3)
 65 |       threepreds++;
 66 |     else if(verdict == 7)
 67 |       sevenpreds++;
 68 |     s.score.zeroGrad(topo); 
 69 |   }
 70 |   double perc = corrects*100.0/(corrects+wrongs);
 71 |   cout<<perc<<"% correct, threes "<<threes<<" sevens "<<sevens<<" threepreds "<<threepreds<<" sevenpreds "<<sevenpreds<<"\n";
 72 | }
 73 | 
 74 | 
 75 | int main()
 76 | {
 77 |   cout<<"Start!"<<endl;
 78 | 
 79 |   MNISTReader mn("gzip/emnist-digits-train-images-idx3-ubyte.gz", "gzip/emnist-digits-train-labels-idx1-ubyte.gz");
 80 |   cout<<"Have "<<mn.num()<<" images"<<endl;
 81 | 
 82 |   MNISTReader mntest("gzip/emnist-digits-test-images-idx3-ubyte.gz", "gzip/emnist-digits-test-labels-idx1-ubyte.gz");
 83 | 
 84 |   vector<int> threeseven;
 85 |   for(unsigned int i = 0 ; i < mn.num(); ++i) {
 86 |     int label = mn.getLabel(i);
 87 |     if(label==3 || label == 7)
 88 |       threeseven.push_back(i);
 89 |   }
 90 |   cout<<"Have "<<threeseven.size()<<" threes and sevens to train with"<<endl;
 91 |   
 92 |   NNArray<float, 1, 28*28> weights;
 93 |   NNArray<float, 1, 1> bias;
 94 |   weights.randomize();
 95 |   bias.randomize();
 96 | 
 97 | 
 98 |   cout<<"Configuring network.. ";
 99 |   cout.flush();
100 | 
101 |   vector<State> states;  
102 |   for(int n=0; n <128; ++n) {
103 |     State s;
104 |     s.init(weights, bias);
105 |     states.push_back(s);
106 |   }
107 |   TrackedFloat totalLoss=0;
108 |   for(auto& s : states)
109 |     totalLoss = totalLoss + s.loss;
110 | 
111 |   cout<<"done: "<< TrackedNumberImp<float>::getCount() <<" nodes"<<endl;
112 |   Batcher batcher(threeseven); // badgerbadger
113 |   auto topo = totalLoss.getTopo();
114 |   for(int tries=0; tries < 100;++tries) {
115 |     if(!(tries % 32))
116 |       scoreModel<State>(weights, bias, mntest);
117 |     
118 |     auto batch = batcher.getBatch(states.size());
119 |     if(batch.empty())
120 |       break;
121 | 
122 |     for(size_t i = 0; i < batch.size(); ++i) {
123 |       State& s = states[i];
124 |       auto idx = batch[i];
125 |       
126 |       mn.pushImage(idx, s.img);
127 |       s.label = mn.getLabel(idx);
128 | 
129 |       if(s.label == 7)
130 |         s.expected = 1;
131 |       if(s.label == 3)
132 |         s.expected = 0;
133 |     }
134 |     
135 |     cout<<"Average loss: ";
136 |     cout.flush();
137 |     cout<<totalLoss.getVal() / states.size() <<endl;
138 |     int corrects=0, wrongs=0;
139 |     int threepreds=0, sevenpreds=0;
140 |     for(auto& s : states) {
141 |       int predicted = s.score.impl->d_val > 0.5 ? 7 : 3; // get the precalculated number
142 |       if(predicted == s.label)
143 |         corrects++;
144 |       else wrongs++;
145 |       if(predicted == 7)
146 |         sevenpreds++;
147 |       else if(predicted ==3)
148 |         threepreds++;
149 |     }
150 |     cout<<"Percent batch correct: "<<100.0*corrects/(corrects+wrongs)<<" threepreds "<<threepreds<<" sevenpreds " <<sevenpreds<<endl;
151 |     totalLoss.backward(topo);
152 |     auto grad = weights.getGrad();
153 |     double lr=0.07;
154 |     /*
155 |     for(int n=0; n < 100; ++n)
156 |       cout<<weights(0,n).getVal()<<" -= "<< (lr*grad(0,n)) <<"\t";
157 |     cout<<endl;
158 |     */
159 |     
160 |     grad *= lr;
161 | 
162 |     weights -= grad;
163 | 
164 |     //weights.zeroGrad();
165 |     
166 |     auto biasgrad = bias.getGrad();
167 |     biasgrad *= lr;
168 |     bias -= biasgrad;
169 |     //    cout<<bias(0,0).getVal() << " -= " << lr*biasgrad(0,0) <<endl;
170 | 
171 |     //    bias.zeroGrad();
172 |     totalLoss.zeroGrad(topo);
173 |   }
174 |   scoreModel<State>(weights, bias, mntest);
175 | }
176 | 
177 | 


--------------------------------------------------------------------------------
/alternate/modular-convo.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <vector>
  3 | #include <fstream>
  4 | #include <memory>
  5 | #include <optional>
  6 | #include "array.hh"
  7 | #include "tracked.hh"
  8 | #include "mnistreader.hh"
  9 | #include "misc.hh"
 10 | #include <string.h>
 11 | 
 12 | #include <fenv.h>
 13 | #include "cnn1.hh"
 14 | #include "cnn-alphabet.hh"
 15 | 
 16 | using TheModel = CNNAlphabetModel<float>;
 17 | 
 18 | using namespace std;
 19 | 
 20 | ofstream g_tree; //("tree.part");
 21 | 
 22 | template<typename M, typename S>
 23 | void scoreModel(S& s, const MNISTReader& mntest, int batchno)
 24 | {
 25 |   unsigned int corrects=0, wrongs=0;
 26 |   static ofstream vcsv("validation.csv");
 27 |   static bool notfirst;
 28 |   if(!notfirst) {
 29 |     vcsv<<"batchno,corperc,avgloss\n";
 30 |     notfirst=true;
 31 |   }
 32 |   
 33 |   M model;
 34 |   model.init(s);
 35 | 
 36 |   auto topo = model.loss.getTopo();
 37 |   Batcher batcher(mntest.num());
 38 |   auto batch = batcher.getBatch(100);
 39 |   double totalLoss=0;
 40 |   for(auto i : batch) {
 41 |     cout<<".";    cout.flush();
 42 |     int label = mntest.getLabel(i) - 1;
 43 | 
 44 |     mntest.pushImage(i, model.img);
 45 |     model.expected.zero();
 46 |     model.expected(0,label) = 1; // "one hot vector"
 47 |     
 48 |     int verdict = model.scores.maxValueIndexOfColumn(0);
 49 | 
 50 |     if(verdict == label) {
 51 |       corrects++;
 52 |     }
 53 |     else {
 54 |       wrongs++;
 55 |     }
 56 |     totalLoss += model.loss.getVal();
 57 |     model.loss.zeroGrad(topo);
 58 |   }
 59 |   cout<<"\n";
 60 |   double perc = corrects*100.0/(corrects+wrongs);
 61 |   double avgLoss = totalLoss/batch.size();
 62 |   cout<<perc<<"% correct, average loss "<<avgLoss<<"\n";
 63 |   vcsv << batchno <<"," << perc << ", " << avgLoss << endl; 
 64 | }
 65 | 
 66 | int main(int argc, char** argv)
 67 | {
 68 |   cout<<"Start!"<<endl;
 69 |   feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
 70 | 
 71 |   string kind = "letters"; // or digits
 72 |   MNISTReader mn("gzip/emnist-"+kind+"-train-images-idx3-ubyte.gz", "gzip/emnist-"+kind+"-train-labels-idx1-ubyte.gz");
 73 |   cout<<"Have "<<mn.num()<<" images"<<endl;
 74 | 
 75 |   MNISTReader mntest("gzip/emnist-"+kind+"-test-images-idx3-ubyte.gz", "gzip/emnist-"+kind+"-test-labels-idx1-ubyte.gz");
 76 | 
 77 |   TheModel::State s;
 78 | 
 79 |   if(argc > 1) {
 80 |     cout<<"Loading model state from "<<argv[1]<<endl;
 81 |     loadModelState(s, argv[1]);
 82 |   }
 83 |   
 84 |   cout<<"Configuring network";
 85 |   cout.flush();
 86 | 
 87 |   vector<TheModel> models;  
 88 |   for(int n=0; n < 22; ++n) {
 89 |     TheModel rm;
 90 |     rm.init(s);
 91 |     models.push_back(rm);
 92 |     cout<<".";
 93 |     cout.flush();
 94 |   }
 95 | 
 96 |   cout<<endl<<"Tying... ";
 97 |   TrackedFloat totalLoss=0;
 98 |   for(auto& m : models)
 99 |     totalLoss = totalLoss + m.loss;
100 |   totalLoss= totalLoss/TrackedFloat(models.size());
101 |   cout<<"done"<<endl;
102 |   cout<<"Getting topology.. ";
103 |   cout.flush();
104 |   auto topo = totalLoss.getTopo();
105 |   cout<<"done"<<endl;
106 |   
107 |   cout<<TrackedNumberImp<float>::getCount()<<" instances"<<endl;
108 |   unsigned int batchno=0;
109 |   ofstream tcsv("training.csv");
110 |   tcsv<<"batchno,corperc,avgloss"<<endl;
111 |   for(;;) {
112 |     Batcher batcher(mn.num()); // badgerbadger
113 |     
114 |     for(int tries=0;;++tries) {
115 |       if(!(tries %32))
116 |         scoreModel<TheModel, TheModel::State>(s, mntest, batchno);
117 |       
118 |       auto batch = batcher.getBatch(models.size());
119 |       if(batch.size() != models.size())
120 |         break;
121 |       vector<int> labels(batch.size());
122 |       for(size_t i = 0; i < batch.size(); ++i) {
123 |         TheModel& m = models.at(i);
124 | 
125 |         auto idx = batch.at(i);
126 |         mn.pushImage(idx, m.img);
127 |         labels[i] = mn.getLabel(idx) -1;
128 |         
129 |         m.expected.zero();
130 |         m.expected(0,labels[i]) = 1; // "one hot vector"
131 |       }
132 |     
133 |       cout<<"Average loss: ";
134 |       cout.flush();
135 |       cout<<totalLoss.getVal() <<". ";
136 |       int corrects=0, wrongs=0;
137 | 
138 |       unsigned int pos=0;
139 |       for(auto& m : models) {
140 |         int predicted = m.scores.maxValueIndexOfColumn(0);        
141 |         if(corrects + wrongs == 0) {
142 |           cout<<"Predicted: "<<predicted<<", actual: "<<labels[pos]<<": ";
143 |           if(predicted == labels[pos])
144 |             cout<<"We got it right!"<<endl;
145 |           else
146 |             cout<<"More learning to do.."<<endl;
147 |           cout<<"Loss: "<<m.loss.getVal()<<", "<<m.scores.flatViewCol()<<endl;
148 |           printImg(m.img);
149 |         }
150 | 
151 |         if(predicted == labels[pos])
152 |           corrects++;
153 |         else wrongs++;
154 |         pos++;
155 |       }
156 |       double perc = 100.0*corrects/(corrects+wrongs);
157 |       cout<<"Percent batch correct: "<<perc<<"%"<<endl;
158 | 
159 |       tcsv << batchno <<"," << perc << ", " << totalLoss.getVal() << endl; 
160 |       
161 |       totalLoss.backward(topo);
162 |       //      cout<<"Done backwarding"<<endl;
163 |       double lr=0.005; // mnist.cpp has 0.01, plus "momentum" of 0.5, which we don't have
164 |       
165 |       s.learn(lr);
166 |       totalLoss.zeroGrad(topo);
167 |       batchno++;
168 | 
169 |       saveModelState(s, "cnn.state");
170 |     }
171 | 
172 |   }
173 |   scoreModel<TheModel>(s, mntest, batchno);
174 | }
175 | 


--------------------------------------------------------------------------------
/tensor-convo.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <vector>
  3 | #include <fstream>
  4 | #include <memory>
  5 | #include <optional>
  6 | #include "tensor2.hh"
  7 | #include "mnistreader.hh"
  8 | #include "misc.hh"
  9 | #include "vizi.hh"
 10 | #include <fenv.h>
 11 | #include "tensor-layers.hh"
 12 | #include "convo-alphabet.hh"
 13 | #include <time.h>
 14 | 
 15 | using namespace std;
 16 | 
 17 | template<typename M, typename S>
 18 | void testModel(SQLiteWriter& sqw, S& s, const MNISTReader& mn, unsigned int startID, int batchno)
 19 | {
 20 |   M m;
 21 |   m.init(s, true); // production
 22 |   
 23 |   Batcher b(mn.num());
 24 |   auto batch = b.getBatch(128);
 25 |   float totalLoss=0;
 26 |   int corrects=0, wrongs=0;
 27 |   
 28 |   auto topo = m.loss.getTopo();
 29 |   DTime dt;
 30 |   dt.start();
 31 |   for(const auto& idx : batch) {
 32 |     m.loss.zerograd(topo);
 33 |     mn.pushImage(idx, m.img);
 34 |     // normalize
 35 |     m.img.normalize(0.172575, 0.25);
 36 |     
 37 |     int label = mn.getLabel(idx) - 1;
 38 |     m.expected.zero();
 39 |     m.expected(0, label) = 1;
 40 | 
 41 |     totalLoss += m.modelloss(0,0); // turns it into a float
 42 |       
 43 |     int predicted = m.scores.maxValueIndexOfColumn(0);
 44 | 
 45 |     if(corrects + wrongs == 0) {
 46 |       printImgTensor(m.img);
 47 |       cout<<"predicted: "<<(char)(predicted+'a')<<", actual: "<<(char)('a'+label)<<", loss: "<<m.modelloss<<"\n";
 48 |     }
 49 |     
 50 |     if(predicted == label)
 51 |       corrects++;
 52 |     else wrongs++;
 53 |   }
 54 |   double perc=100.0*corrects/(corrects+wrongs);
 55 |   cout<<"Validation batch average loss: "<<totalLoss/batch.size()<<", percentage correct: "<<perc<<", took "<<dt.lapUsec()/1000<<" ms for "<<batch.size()<<" images\n";
 56 |   
 57 |   sqw.addValue({
 58 |       {"startID", startID},
 59 |       {"batchno", batchno},
 60 |       {"cputime", (double) clock()/CLOCKS_PER_SEC},
 61 |       {"corperc", perc},
 62 |       {"avgloss", totalLoss/batch.size()}}, "validation");
 63 | }
 64 | 
 65 | int main(int argc, char **argv)
 66 | {
 67 |   feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
 68 | 
 69 |   MNISTReader mn("gzip/emnist-letters-train-images-idx3-ubyte.gz", "gzip/emnist-letters-train-labels-idx1-ubyte.gz");
 70 |   MNISTReader mntest("gzip/emnist-letters-test-images-idx3-ubyte.gz", "gzip/emnist-letters-test-labels-idx1-ubyte.gz");
 71 | 
 72 |   cout<<"Have "<<mn.num()<<" training images and "<<mntest.num()<<" test images"<<endl;
 73 | 
 74 |   ConvoAlphabetModel m;
 75 |   ConvoAlphabetModel::State s;
 76 | 
 77 |   if(argc==2) {
 78 |     cout<<"Loading model state from file '"<<argv[1]<<"'\n";
 79 |     loadModelState(s, argv[1]);
 80 |   }
 81 |   else
 82 |     s.randomize();
 83 |   
 84 |   m.init(s);
 85 | 
 86 |   auto topo = m.loss.getTopo();
 87 |   cout<<"Topo.size(): "<<topo.size()<<endl;
 88 | 
 89 |   SQLiteWriter sqw("convo-vals.sqlite3");
 90 |   int64_t startID=time(0);
 91 |   
 92 |   int batchno = 0;
 93 | 
 94 |   for(;;) {
 95 |     Batcher batcher(mn.num());
 96 | 
 97 |     DTime dt;
 98 |     for(unsigned int tries = 0 ;; ++tries) {
 99 |       auto batch = batcher.getBatch(64);
100 |       if(batch.empty())
101 |         break;
102 | 
103 |       if(!(tries % 32)) {
104 |         testModel<ConvoAlphabetModel>(sqw, s, mntest, startID, batchno);
105 |         saveModelState(s, "tensor-convo.state");
106 |       }
107 |       if(batchno < 32 || !(tries%32)) {
108 |         s.emit(sqw, startID, batchno, batch.size());
109 |       }
110 |       dt.start();
111 |       
112 |       batchno++;
113 | 
114 |       float totalLoss = 0, totalWeightsLoss=0;
115 |       unsigned int corrects=0, wrongs=0;
116 |       
117 |       m.loss.zeroAccumGrads(topo);
118 | 
119 |       for(const auto& idx : batch) {     
120 |         mn.pushImage(idx, m.img);
121 |         // normalize
122 |         m.img.normalize(0.172575, 0.25);
123 |         
124 |         int label = mn.getLabel(idx) -1; // they count from 1 over at NIST!
125 |         m.expected.oneHotColumn(label);
126 |         
127 |         totalLoss += m.modelloss(0,0); // turns it into a float
128 |         totalWeightsLoss += m.weightsloss(0,0);
129 |         int predicted = m.scores.maxValueIndexOfColumn(0);
130 |         
131 |         if(corrects + wrongs == 0) {
132 |           cout<<"predicted: "<<(char)(predicted+'a')<<", actual: "<<(char)(label+'a')<<", loss: "<<m.modelloss<<"\n";
133 |           printImgTensor(m.img);
134 |         }
135 |         
136 |         if(predicted == label)
137 |           corrects++;
138 |         else wrongs++;
139 |         
140 |         // backward the thing
141 |         m.loss.backward(topo);
142 |         m.loss.accumGrads(topo); 
143 |         // clear grads & havevalue
144 |         m.loss.zerograd(topo);
145 |       }
146 |       double perc = 100.0*corrects/(corrects+wrongs);
147 |       cout<<"Batch "<<batchno<<" average loss " << totalLoss/batch.size()<<", weightsloss " <<totalWeightsLoss/batch.size()<<", percent batch correct "<<perc<<"%, "<<dt.lapUsec()/1000<<"ms/batch"<<endl;
148 | 
149 |       double lr=0.010 / batch.size(); // 0.010 works well at the beginning
150 |       double momentum = 0.9;
151 |       s.learn(lr, momentum);
152 | 
153 |       // tcsv<<"batchno,cputime,corperc,avgloss,batchsize,lr,momentum"<<endl;
154 |       sqw.addValue({
155 |           {"startID", startID}, {"batchno", batchno},   {"cputime", (double)clock()/CLOCKS_PER_SEC},
156 |           {"corperc", perc},    {"avgloss", totalLoss/batch.size()},
157 |           {"batchsize", (int)batch.size()}, {"lr", lr*batch.size()}, {"momentum", momentum}}, "training");
158 |     }
159 |   }
160 | }
161 | 
162 | 


--------------------------------------------------------------------------------
/alternate/first-relu.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <vector>
  3 | #include <fstream>
  4 | #include <memory>
  5 | #include <optional>
  6 | #include "array.hh"
  7 | #include "tracked.hh"
  8 | #include "mnistreader.hh"
  9 | #include "misc.hh"
 10 | #include <fenv.h>
 11 | 
 12 | using namespace std;
 13 | 
 14 | ofstream g_tree;//("tree.part");
 15 | 
 16 | struct ReluModel {
 17 |   NNArray<float, 28, 28> img;
 18 | 
 19 |   int label;
 20 |   NNArray<float, 10, 1> scores;
 21 |   NNArray<float, 1, 10> expected;
 22 | 
 23 |   TrackedFloat loss;
 24 | 
 25 |   struct State
 26 |   {
 27 |     //             out  in
 28 |     NNArray<float, 128, 28*28> w1;
 29 |     NNArray<float, 128, 1> b1;
 30 |     
 31 |     NNArray<float, 64, 128> w2;
 32 |     NNArray<float, 64, 1> b2;
 33 | 
 34 |     NNArray<float, 10, 64> w3;
 35 |     NNArray<float, 10, 1> b3;
 36 | 
 37 |   };
 38 |   
 39 |   void init(State& s)
 40 |   {
 41 |     img.zero();
 42 |     auto flat =    img.flatViewRow();
 43 | 
 44 |     auto output =  s.w1 * flat + s.b1;
 45 | 
 46 |     auto output2 = output.applyFunc(ReluFunc());
 47 | 
 48 |     auto output3 = s.w2 * output2 + s.b2;
 49 |     auto output4 = output3.applyFunc(ReluFunc());
 50 | 
 51 |     auto output5 = s.w3 * output4 + s.b3;
 52 | 
 53 |     scores = output5.logSoftMax();
 54 |     expected.zero();
 55 |     loss = TrackedFloat(0)-(expected*scores)(0,0);
 56 |   }
 57 | };
 58 | 
 59 | template<typename M, typename S>
 60 | void scoreModel(S& s, const MNISTReader& mntest)
 61 | {
 62 |   unsigned int corrects=0, wrongs=0;
 63 | 
 64 |   M model;
 65 |   model.init(s);
 66 | 
 67 |   unsigned int limit = mntest.num() - 1;
 68 |   limit=100;
 69 |   for(unsigned int i = 0 ; i < limit; ++i){
 70 |     cout<<".";
 71 |     cout.flush();
 72 |     int label = mntest.getLabel(i);
 73 | 
 74 |     mntest.pushImage(i, model.img);
 75 |     
 76 |     int verdict = model.scores.maxValueIndexOfColumn(0);
 77 |     //    cout<<"label "<<(int)label<<" result(0) "<<result(0)<<" verdict " <<verdict<<" pic "<<pic.mean()<<endl;
 78 |     //cout<<"l1bias "<<l.l1.bias<<" l2bias "<<l.l2.bias<<endl;
 79 |     //    cout<<l.l1.weights.cwiseAbs().mean()<<endl;
 80 |     if(verdict == label) {
 81 |       corrects++;
 82 |     }
 83 |     else {
 84 |       wrongs++;
 85 |     }
 86 |   }
 87 |   cout<<"\n";
 88 |   double perc = corrects*100.0/(corrects+wrongs);
 89 |   cout<<perc<<"% correct\n";
 90 | }
 91 | 
 92 | 
 93 | int main()
 94 | {
 95 |   cout<<"Start!"<<endl;
 96 |   feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
 97 | 
 98 |   MNISTReader mn("gzip/emnist-digits-train-images-idx3-ubyte.gz", "gzip/emnist-digits-train-labels-idx1-ubyte.gz");
 99 |   MNISTReader mntest("gzip/emnist-digits-test-images-idx3-ubyte.gz", "gzip/emnist-digits-test-labels-idx1-ubyte.gz");
100 | 
101 |   cout<<"Have "<<mn.num()<<" training images and "<<mntest.num()<<" test images"<<endl;
102 |   
103 |   ReluModel::State s;
104 |   s.w1.randomize(sqrt(1/(28.0*28.0)));
105 |   s.b1.randomize(sqrt(1/(28.0*28.0)));
106 |   
107 |   s.w2.randomize(sqrt(1/(128.0)));
108 |   s.b2.randomize(sqrt(1/(128.0)));
109 |                  
110 |   s.w3.randomize(sqrt(1/(64.0)));
111 |   s.b3.randomize(sqrt(1/(64.0)));
112 | 
113 | 
114 |   cout<<"Configuring network";
115 |   cout.flush();
116 | 
117 |   vector<ReluModel> models;  
118 |   for(int n=0; n <256; ++n) {
119 |     ReluModel rm;
120 |     rm.init(s);
121 |     models.push_back(rm);
122 |     cout<<".";
123 |     cout.flush();
124 |   }
125 | 
126 |   cout<<endl<<"Tying... ";
127 |   TrackedFloat totalLoss=0;
128 |   for(auto& m : models)
129 |     totalLoss = totalLoss + m.loss;
130 |   totalLoss = totalLoss/TrackedFloat(models.size());
131 |   cout<<"done"<<endl;
132 |   cout<<"Getting topology.. ";
133 |   cout.flush();
134 |   auto topo = totalLoss.getTopo();
135 |   cout<<"done"<<endl;
136 |   
137 |   cout<<TrackedNumberImp<float>::getCount()<<" instances"<<endl;
138 |   Batcher batcher(mn.num()); // badgerbadger
139 |  
140 |   for(int tries=0;;++tries) {
141 |     
142 |     if(!(tries %32))
143 |       scoreModel<ReluModel, ReluModel::State>(s, mntest);
144 | 
145 |     
146 |     auto batch = batcher.getBatch(models.size());
147 |     if(batch.empty())
148 |       break;
149 |     for(size_t i = 0; i < batch.size(); ++i) {
150 |       ReluModel& m = models.at(i);
151 | 
152 |       auto idx = batch.at(i);
153 | 
154 |       mn.pushImage(idx, m.img);
155 | 
156 |       m.label = mn.getLabel(idx);
157 | 
158 |       m.expected.zero();
159 |       m.expected(0,m.label) = 1;
160 |     }
161 |     
162 |     cout<<"Average loss: ";
163 |     cout.flush();
164 |     cout<<totalLoss.getVal() <<". ";
165 |     int corrects=0, wrongs=0;
166 | 
167 |     for(auto& m : models) {
168 |       if(corrects + wrongs == 0) {
169 |         cout<<m.expected<<endl;
170 |         cout<<m.scores<<endl;
171 |         cout<<"Loss: "<<m.loss.getVal()<<endl;
172 |       }
173 | 
174 |       int predicted = m.scores.maxValueIndexOfColumn(0);
175 |       if(predicted == m.label)
176 |         corrects++;
177 |       else wrongs++;
178 |     }
179 |     cout<<"Percent batch correct: "<<100.0*corrects/(corrects+wrongs)<<"%"<<endl;
180 |     totalLoss.backward(topo);
181 |     cout<<"Done backwarding"<<endl;
182 | 
183 |     double lr=0.2;//.2;
184 | 
185 |     auto doLearn=[&lr](auto& v){
186 |       auto grad = v.getGrad();
187 |       grad *= lr;
188 |       v -= grad;
189 |     };
190 | 
191 |     doLearn(s.w1);
192 |     doLearn(s.w2);
193 |     doLearn(s.w3);
194 |     doLearn(s.b1);
195 |     doLearn(s.b2);
196 |     doLearn(s.b3);
197 |     
198 |     cout<<"Resetting grads"<<endl;
199 |     totalLoss.zeroGrad(topo);
200 |   }
201 |   scoreModel<ReluModel>(s, mntest);
202 | }
203 | 
204 | 


--------------------------------------------------------------------------------
/trackedfuncs.hh:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include "fvector.hh"
  3 | 
  4 | struct FuncStruct
  5 | {
  6 |   typedef float (*floatfunc_t)(const float& f);
  7 |   typedef fvector<4>(*fvect4func_t)(const fvector<4>& f);
  8 |   typedef fvector<8>(*fvect8func_t)(const fvector<8>& f);
  9 |   floatfunc_t func;
 10 |   floatfunc_t deriv;
 11 |   fvect4func_t func4;
 12 |   fvect4func_t deriv4;
 13 |   fvect8func_t func8;
 14 |   fvect8func_t deriv8;
 15 |   std::string name;
 16 |   
 17 |   template<typename T>
 18 |   using retfunc = T(*)(const T&);
 19 | };
 20 | 
 21 | template<typename T> // primary template
 22 | FuncStruct::retfunc<T> getFunc(const FuncStruct&);
 23 | 
 24 | template<>
 25 | inline
 26 | FuncStruct::retfunc<float> getFunc(const FuncStruct& fs)
 27 | {
 28 |   return fs.func;
 29 | }
 30 | 
 31 | template<>
 32 | inline
 33 | FuncStruct::retfunc<fvector<4>> getFunc(const FuncStruct& fs)
 34 | {
 35 |   return fs.func4;
 36 | }
 37 | 
 38 | template<> inline
 39 | FuncStruct::retfunc<fvector<8>> getFunc(const FuncStruct& fs)
 40 | {
 41 |   return fs.func8;
 42 | }
 43 | 
 44 | template<typename T> // primary template
 45 | FuncStruct::retfunc<T> getDeriv(const FuncStruct&);
 46 | 
 47 | template<> inline
 48 | FuncStruct::retfunc<float> getDeriv(const FuncStruct& fs)
 49 | {
 50 |   return fs.deriv;
 51 | }
 52 | 
 53 | template<> inline
 54 | FuncStruct::retfunc<fvector<4>> getDeriv(const FuncStruct& fs)
 55 | {
 56 |   return fs.deriv4;
 57 | }
 58 | 
 59 | 
 60 | template<> inline
 61 | FuncStruct::retfunc<fvector<8>> getDeriv(const FuncStruct& fs)
 62 | {
 63 |   return fs.deriv8;
 64 | }
 65 | 
 66 | inline FuncStruct MakeExpFunc()
 67 | {
 68 |   FuncStruct ret;
 69 |   ret.func = [](const float& f) -> float { return expf(f);};
 70 |   ret.deriv = [](const float& f) -> float { return expf(f);};
 71 |   ret.func4 = [](const fvector<4>& f) { return exp(f);};
 72 |   ret.deriv4 = [](const fvector<4>& f) -> fvector<4> { return exp(f);};
 73 |   ret.func8 = [](const fvector<8>& f) { return exp(f);};
 74 |   ret.deriv8 = [](const fvector<8>& f) -> fvector<8> { return exp(f);};
 75 |   ret.name="exp";
 76 |   return ret;
 77 | }
 78 | 
 79 | inline FuncStruct MakeSquareFunc()
 80 | {
 81 |   FuncStruct ret;
 82 |   ret.func = [](const float& f) -> float { return f*f;};
 83 |   ret.deriv = [](const float& f) -> float { return 2*f;};
 84 |   ret.func4 = [](const fvector<4>& f) { return f*f;};
 85 |   ret.deriv4 = [](const fvector<4>& f) -> fvector<4> { return 2*f;};
 86 |   ret.func8 = [](const fvector<8>& f) { return f*f;};
 87 |   ret.deriv8 = [](const fvector<8>& f) -> fvector<8> { return 2*f;};
 88 |   ret.name="square";
 89 |   return ret;
 90 | }
 91 | 
 92 | inline FuncStruct MakeReluFunc()
 93 | {
 94 |   FuncStruct ret;
 95 |   ret.func = [](const float& f) -> float { return std::max(0.0F, f); };
 96 |   ret.deriv = [](const float& f) -> float { return f < 0.0 ? 0.0F : 1.0F; };
 97 |   ret.func4 = [](const fvector<4>& in) {
 98 |     fvector<4> cmp = 0;
 99 |     return (cmp < in) * in;
100 |   };
101 |   ret.deriv4 = [](const fvector<4>& in) {
102 |     fvector<4> cmp = 0;
103 |     return (cmp < in) * 1.0;
104 |   };
105 |   ret.func8 = [](const fvector<8>& in) {
106 |     fvector<8> cmp = 0;
107 |     return (cmp < in) * in;
108 |   };
109 |   ret.deriv8 = [](const fvector<8>& in) {
110 |     fvector<8> cmp = 0;
111 |     return (cmp < in) * 1.0;
112 |   };
113 |   ret.name="relu";
114 |   return ret;
115 | }
116 | 
117 | 
118 | inline FuncStruct MakeTanhFunc()
119 | {
120 |   FuncStruct ret;
121 |   ret.func = [](const float& in) { return tanhf(in);  };
122 |   ret.deriv = [](const float& in)  {  float t = tanh(in); return 1-t*t;  };
123 | 
124 |   ret.func8 = [](const fvector<8>& in) { return tanh(in); };
125 |   ret.deriv8 =[](const fvector<8>& in) { fvector<8> t = tanh(in); return fvector<8>(1.0) - t*t;  };
126 |   ret.name = "tanh";
127 |   return ret;
128 | };
129 | 
130 | 
131 | inline FuncStruct MakeSigmoidFunc()
132 | {
133 |   FuncStruct ret;
134 |   ret.func8 = [](const fvector<8>& in)
135 |   {
136 |     return 1.0F / (fvector<8>(1.0F) + exp(-in));
137 |   };
138 |   ret.deriv8 = [](const fvector<8>& in)
139 |   {
140 |     fvector<8> sigma = 1.0F / (fvector<8>(1.0F) + exp(-in));
141 |     return sigma * (fvector<8>(1.0F) - sigma);
142 |   };
143 | 
144 |   ret.func = [](const float& in)
145 |   {
146 |     return 1.0F / (1.0F + expf(-in));
147 |   };
148 |   ret.deriv = [](const float& in)
149 |   {
150 |     float sigma = 1.0F / (1.0F + expf(-in));
151 |     return sigma * (1.0F - sigma);
152 |   };
153 |   ret.name="sigmoid";
154 |   return ret;
155 | };
156 | 
157 | 
158 | inline FuncStruct MakeLogFunc()
159 | {
160 |   struct FuncStruct ret;
161 |   
162 |   ret.func = [](const float& in)
163 |   {
164 |     if (in == 0.0F)
165 |       return -80.0F;
166 |     return logf(in);
167 |   };
168 |   ret.deriv = [](const float& in)
169 |   {
170 |     if (in == 0.0F)
171 |       return 80.0F;
172 |     return 1.0F / in;
173 |   };
174 | 
175 |   ret.func8 = [](const fvector<8>& in)
176 |   {
177 |     return log(in);
178 |   };
179 |   ret.deriv8 = [](const fvector<8>& in)
180 |   {
181 |     return 1.0/in;
182 |   };
183 |   
184 |   ret.name = "log";
185 |   return ret;
186 | };
187 | 
188 | static std::vector<FuncStruct> g_fss({
189 |     MakeSigmoidFunc(), // 0
190 |     MakeReluFunc(),    // 1
191 |     MakeExpFunc(),     // 2 
192 |     MakeLogFunc(),     // 3
193 |     MakeTanhFunc(),    // 4
194 |     MakeSquareFunc()});// 5
195 | 
196 | inline uint8_t SigmoidFunc(){ return 0; }
197 | inline uint8_t ReluFunc(){ return 1; }
198 | inline uint8_t ExpFunc(){ return 2; }
199 | inline uint8_t LogFunc(){ return 3; }
200 | inline uint8_t TanhFunc(){ return 4; }
201 | inline uint8_t SquareFunc(){ return 5; }
202 | 
203 | 
204 | 


--------------------------------------------------------------------------------
/alternate/modular-linear-convo.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <vector>
  3 | #include <fstream>
  4 | #include <memory>
  5 | #include <optional>
  6 | #include "array.hh"
  7 | #include "tracked.hh"
  8 | #include "mnistreader.hh"
  9 | #include "misc.hh"
 10 | #include <string.h>
 11 | 
 12 | #include <fenv.h>
 13 | #include "cnn1.hh"
 14 | #include "cnn-alphabet.hh"
 15 | #include "fvector.hh"
 16 | 
 17 | using TheModel = CNNAlphabetModel<fvector<8>>;
 18 | 
 19 | using namespace std;
 20 | 
 21 | ofstream g_tree; //("tree.part");
 22 | 
 23 | template<typename M, typename S>
 24 | void scoreModel(S& s, const MNISTReader& mntest, int batchno)
 25 | {
 26 |   unsigned int corrects=0, wrongs=0;
 27 |   static ofstream vcsv("validation2.csv");
 28 |   static bool notfirst;
 29 |   if(!notfirst) {
 30 |     vcsv<<"batchno,corperc,avgloss\n";
 31 |     notfirst=true;
 32 |   }
 33 |   
 34 |   M mod;
 35 |   mod.init(s);
 36 | 
 37 |   auto topo = mod.loss.getTopo();
 38 |   Batcher batcher(mntest.num());
 39 |   auto batch = batcher.getBatch(80);
 40 |   
 41 |   fvector<8> totLoss=0.0; 
 42 |   
 43 |   while(!batch.empty()) {
 44 |     mod.expected.zero();
 45 |     fvector<8> label = 0;
 46 |     for(int n = 0; n < 8 ; ++n) {
 47 |       auto idx = batch.front();
 48 |       batch.pop_front();
 49 |       
 50 |       mntest.pushImage(idx, mod.img, n);
 51 |       label.v[n] = mntest.getLabel(idx) - 1;  // a == 1..
 52 |       
 53 |       mod.expected(0, label.v[n]).impl->d_val.v[n] = 1; // "one hot vector"
 54 |     }
 55 |     
 56 |     totLoss += mod.loss.getVal();
 57 |     
 58 |     for(int n = 0; n < 8; ++n) {
 59 |       int verdict = mod.scores.getUnparallel(n).maxValueIndexOfColumn(0);
 60 |       if(verdict == label.v[n])
 61 |         corrects++;
 62 |       else
 63 |         wrongs++;
 64 |     }
 65 |     mod.loss.zeroGrad(topo);
 66 |   }
 67 |   double avgLoss = totLoss.sum()/80.0;
 68 |   double perc = 100.0*corrects/(corrects+wrongs);
 69 |   cout<<perc<<"% correct, average loss "<<avgLoss<<"\n";
 70 |   vcsv << batchno <<"," << perc << ", " << avgLoss << endl; 
 71 | }
 72 | 
 73 | int main(int argc, char** argv)
 74 | {
 75 |   feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
 76 | 
 77 |   string kind = "letters"; // or digits
 78 |   MNISTReader mn("gzip/emnist-"+kind+"-train-images-idx3-ubyte.gz", "gzip/emnist-"+kind+"-train-labels-idx1-ubyte.gz");
 79 | 
 80 |   MNISTReader mntest("gzip/emnist-"+kind+"-test-images-idx3-ubyte.gz", "gzip/emnist-"+kind+"-test-labels-idx1-ubyte.gz");
 81 | 
 82 |   cout<<"Have "<<mn.num()<<" images for training, "<<mntest.num()<<" for validation"<<endl;
 83 |   
 84 |   TheModel::State s;
 85 | 
 86 |   if(argc > 1) {
 87 |     cout<<"Loading model state from "<<argv[1]<<endl;
 88 |     loadModelState(s, argv[1]);
 89 |   }
 90 | 
 91 |   TheModel mod;
 92 |   mod.init(s);
 93 | 
 94 |   cout<<"Getting topo"<<endl;
 95 |   auto topo = mod.loss.getTopo();
 96 |   
 97 |   cout<<TrackedNumberImp<float>::getCount()<<" instances, model has "<<s.size()<<" parameters, "<<topo.size()<<" nodes to be visited\n";
 98 | 
 99 |   unsigned int batchno=0;
100 |   ofstream tcsv("training2.csv");
101 |   tcsv<<"batchno,corperc,avgloss"<<endl;
102 | 
103 |   constexpr int batchsize=64;
104 | 
105 |   for(;;) {
106 |     Batcher batcher(mn.num()); // badgerbadger
107 | 
108 |     for(int tries=0;;++tries) {
109 |       if(!((tries+0) %32))
110 |         scoreModel<TheModel, TheModel::State>(s, mntest, batchno);
111 |       DTime dt;
112 |       dt.start();
113 |       auto batch = batcher.getBatch(batchsize);
114 |       if(batch.size() != batchsize)
115 |         break;
116 | 
117 |       fvector<8> totLoss=0.0; 
118 |       int corrects=0, wrongs=0;
119 | 
120 |       TheModel::State gather;
121 |       gather.zeroGrad();
122 |       
123 |       while(!batch.empty()) {
124 |         mod.expected.zero();
125 |         fvector<8> label=0;
126 |         for(int n = 0; n < 8 ; ++n) {
127 |           auto idx = batch.front();
128 |           batch.pop_front();
129 |           
130 |           mn.pushImage(idx, mod.img, n);
131 |           label.v[n] = mn.getLabel(idx) - 1;  // a == 1..
132 |           
133 |           mod.expected(0, label.v[n]).impl->d_val.v[n] = 1; // "one hot vector"
134 |         }
135 | 
136 |         totLoss += mod.loss.getVal();
137 |         mod.loss.backward(topo);
138 | 
139 |         gather.addGrad(s);
140 | 
141 |         for(int n = 0; n < 8; ++n) {
142 |           int verdict = mod.scores.getUnparallel(n).maxValueIndexOfColumn(0);
143 | 
144 |           if(corrects + wrongs == 0) {
145 |             cout<<"Predicted: '"<< (char)('a'+verdict)<<"', actual: '"<< (char)('a'+label.v[n]) <<"': ";
146 |             if(verdict == label.v[n])
147 |               cout<<"We got it right!"<<endl;
148 |             else
149 |               cout<<"More learning to do.."<<endl;
150 |             cout<<"Loss: "<<mod.loss.getVal().v[n]<<", "<<mod.scores.getUnparallel(n).flatViewCol()<<endl;
151 |             printImg(mod.img.getUnparallel(n));
152 |           }
153 |           
154 |           if(verdict == label.v[n])
155 |             corrects++;
156 |           else
157 |             wrongs++;
158 |         }
159 | 
160 |         mod.loss.zeroGrad(topo);
161 |       }
162 | 
163 |       double perc = 100.0*corrects/(corrects+wrongs);
164 |       cout<<"Percent batch correct: "<<perc<<"%, average loss: "<<totLoss.sum()/batchsize <<endl;
165 | 
166 |       tcsv << batchno <<"," << perc << ", " << totLoss.sum() / batchsize << endl; 
167 |       
168 |       double lr=0.005; // mnist.cpp has 0.01, plus "momentum" of 0.5, which we don't have
169 |       s.setGrad(gather, batchsize);
170 |       s.learn(lr);
171 |       batchno++;
172 |       cout<<dt.lapUsec()/1000000.0<<" seconds/batch of " << batchsize << endl;;
173 |       saveModelState(s, "cnn-lin.state");
174 |     }
175 |   }
176 | }
177 | 


--------------------------------------------------------------------------------
/tensor-gru.cc:
--------------------------------------------------------------------------------
  1 | #include "gru-layer.hh"
  2 | #include "textsupport.hh"
  3 | #include "misc.hh"
  4 | 
  5 | using namespace std;
  6 | 
  7 | template<typename T, unsigned int SYMBOLS>
  8 | struct GRUModel 
  9 | {
 10 |   struct State : ModelState<T>
 11 |   {
 12 |     //          IN   HIDDEN
 13 |     GRULayer<T, SYMBOLS, 256> gm1;
 14 |     GRULayer<T, 256, 256> gm2;
 15 |     GRULayer<T, 256, 256> gm3;
 16 |     Linear<T, 256, SYMBOLS> fc;
 17 | 
 18 |     State()
 19 |     {
 20 |       this->d_members = {{&gm1, "gm1"}, {&gm2, "gm2"}, {&gm3, "gm3"}, {&fc, "fc"}};
 21 |     }
 22 |     
 23 |   };
 24 |   vector<Tensor<T>> invec;   // SYMBOLS  1
 25 |   vector<Tensor<T>> expvec;  // 1  SYMBOLS
 26 |   vector<Tensor<T>> scorevec;// SYMBOLS  1
 27 | 
 28 |   Tensor<T> totloss;
 29 |   
 30 |   void unroll(State& s, unsigned int choplen)
 31 |   {
 32 |     cout<<"Unrolling the GRU";
 33 |     totloss= Tensor<T>(1, 1);
 34 |     totloss(0,0) = 0.0;
 35 |     Tensor<T> choplent(1,1);
 36 |     choplent(0,0) = choplen;
 37 | 
 38 |     for(size_t i = 0 ; i < choplen; ++i) {
 39 |       cout<<"."; cout.flush();
 40 |       Tensor<T> in(SYMBOLS, 1);
 41 |       Tensor<T> expected(1,SYMBOLS);
 42 |       in.zero();  
 43 |       expected.zero(); 
 44 |       
 45 |       invec.push_back(in);
 46 |       expvec.push_back(expected);
 47 |       auto res1 = s.fc.forward(s.gm3.forward(s.gm2.forward(s.gm1.forward(in))));
 48 |       auto score = makeLogSoftMax(res1);
 49 |       scorevec.push_back(score);
 50 |       auto loss =  -(expected*score);
 51 |       totloss = totloss + loss;
 52 |     }
 53 |     totloss = totloss/choplent; // otherwise the gradient is too high
 54 |     cout<<"\n";
 55 |   }
 56 | };
 57 | 
 58 | int sampleWithTemperature(const Tensor<float>::EigenMatrix& in, float t [[maybe_unused]] = 0)
 59 | {
 60 |   std::random_device rd;
 61 |   std::mt19937 gen(rd());
 62 |   vector<double> ref;
 63 |   for(int r = 0; r< in.rows();++r)
 64 |     for(int c = 0; c< in.cols();++c)
 65 |       ref.push_back(exp(3.0*in(r,c)));
 66 | 
 67 |   std::discrete_distribution<> d(ref.begin(), ref.end());
 68 |   int pick = d(gen);
 69 | 
 70 |   return pick;
 71 | }
 72 | 
 73 | int main(int argc, char **argv)
 74 | {
 75 |   constexpr int tokens = 95;
 76 |   BiMapper bm("corpus.txt",tokens);
 77 |   constexpr int choplen= 75;
 78 |   vector<string> sentences=textChopper("corpus.txt", choplen, 10);
 79 |   cout<<"Got "<<sentences.size()<<" sentences"<<endl;
 80 | 
 81 |   GRUModel<float, tokens> grum;
 82 |   GRUModel<float, tokens>::State s;
 83 | 
 84 |   vector<char> charset;
 85 |   for(int n=0; n < tokens;++n) {
 86 |     cout<<bm.i2c(n);
 87 |     charset.push_back(bm.i2c(n));
 88 |   }
 89 |   cout<<endl;
 90 |   sort(charset.begin(), charset.end());
 91 |   for(const auto& c: charset) {
 92 |     cout<<c;
 93 |   }
 94 |   cout<<endl;
 95 |   if(argc > 2) {
 96 |     int len = 1000;
 97 |     grum.unroll(s, len);
 98 |     cout<<"Loading state from "<<argv[1]<<endl;
 99 |     loadModelState(s, argv[1]);
100 |     auto topo = grum.scorevec[0].getTopo();
101 |     int res;
102 |     int n;
103 |     for(n=0; n < strlen(argv[2]); ++n) {
104 |       grum.invec[n].oneHotRow(bm.c2i(argv[2][n])); // "prompt"
105 |       grum.scorevec[n](0,0);
106 | 
107 |       res = grum.scorevec[n].maxValueIndexOfColumn(0);
108 |       cout << bm.i2c(res);
109 |     }
110 |     cout<<endl;
111 |     for(; n < len ; ++n) {
112 |       grum.invec[n].oneHotRow(res);      
113 |       grum.scorevec[n](0,0);
114 |       res = sampleWithTemperature(grum.scorevec[n].d_imp->d_val);
115 |       //      res = grum.scorevec[n].maxValueIndexOfColumn(0);
116 |       cout << bm.i2c(res);
117 |     }
118 |     cout<<endl;
119 |     return 0;
120 |   }
121 |   
122 |   grum.unroll(s, choplen - 1);
123 | 
124 |   // is now randomized
125 |   if(argc>1) {
126 |     cout<<"Loading state from "<<argv[1]<<endl;
127 |     loadModelState(s, argv[1]);
128 |   }
129 |   
130 |   constexpr int batchsize = 100;
131 | 
132 |   auto topo = grum.totloss.getTopo();
133 |   cout<<"Topo has "<< topo.size() <<" entries"<<endl;
134 |   unsigned int batchno = 0;
135 |   cout<<"Starting the work"<<endl;
136 |   for(;;) {
137 |     Batcher batcher(sentences.size());
138 |     for(;;) { // the batch loop
139 |       auto batch = batcher.getBatch(batchsize);
140 |       if(batch.empty())
141 |         break;
142 |       float batchloss = 0;
143 |       ++batchno;
144 |       grum.totloss.zeroAccumGrads(topo);
145 |       for(const auto& idx : batch) {
146 |         s.gm1.d_origprevh.zero();
147 |         s.gm2.d_origprevh.zero();
148 |         s.gm3.d_origprevh.zero();
149 | 
150 |         //        cout << "s.gm1.d_prevh:\n"<< s.gm1.d_prevh<<endl;
151 |         //cout << "s.gm1.d_origprevh:\n"<< s.gm1.d_origprevh<<endl;
152 |         
153 |         string input = sentences[idx];
154 |         std::string output;
155 | 
156 |         cout<<"IN:  ";
157 |         for(size_t pos = 0 ; pos < input.size() - 1; ++pos) {
158 |           cout<<input.at(pos);
159 |           grum.invec[pos].zero();
160 |           grum.invec[pos](bm.c2i(input.at(pos)), 0) = 1.0;
161 |           grum.expvec[pos].zero();
162 |           grum.expvec[pos](0, bm.c2i(input.at(pos+1)))= 1.0;
163 |         }
164 |         cout<<"\nOUT:  ";
165 |         batchloss += grum.totloss(0,0); // triggers the calculation
166 |         for(size_t pos = 0 ; pos < choplen - 1; ++pos) {
167 |           cout<< bm.i2c(grum.scorevec[pos].maxValueIndexOfColumn(0));
168 |         }
169 |         cout<<"\n";
170 |         grum.totloss.backward(topo);
171 |         grum.totloss.accumGrads(topo);
172 |         grum.totloss.zerograd(topo);
173 |       }
174 |       
175 |       batchloss /= batchsize;
176 |       
177 |       float lr=0.001/batchsize;
178 |       cout<<"Average batch loss: "<<batchloss<<endl;
179 |       s.learnAdam(1.0/batchsize, batchno, 0.001); // 0.001 is start
180 |       batchno++;
181 |       cout<<"\n\n";
182 |       saveModelState(s, "tensor-gru.state");
183 |     }
184 |   }
185 | }
186 | 
187 | 


--------------------------------------------------------------------------------
/ui.cc:
--------------------------------------------------------------------------------
  1 | #include "misc.hh"
  2 | #include "imgui.h"
  3 | #include "imgui_impl_glfw.h"
  4 | #include "imgui_impl_opengl3.h"
  5 | #include <implot.h>
  6 | #include <stdio.h>
  7 | #define GL_SILENCE_DEPRECATION
  8 | #if defined(IMGUI_IMPL_OPENGL_ES2)
  9 | #include <GLES2/gl2.h>
 10 | #endif
 11 | #include <GLFW/glfw3.h> // Will drag system OpenGL headers
 12 | #include <iostream>
 13 | #include "tracked.hh"
 14 | using namespace std;
 15 | 
 16 | 
 17 | 
 18 | 
 19 | static void glfw_error_callback(int error, const char* description)
 20 | {
 21 |   fprintf(stderr, "Glfw Error %d: %s\n", error, description);
 22 | }
 23 | 
 24 | int graphicsThread()
 25 | {
 26 |   glfwSetErrorCallback(glfw_error_callback);
 27 |   if (!glfwInit())
 28 |     return 1;
 29 | 
 30 |   // Decide GL+GLSL versions
 31 | #if defined(IMGUI_IMPL_OPENGL_ES2)
 32 |   // GL ES 2.0 + GLSL 100
 33 |   const char* glsl_version = "#version 100";
 34 |   glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 2);
 35 |   glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 0);
 36 |   glfwWindowHint(GLFW_CLIENT_API, GLFW_OPENGL_ES_API);
 37 | #elif defined(__APPLE__)
 38 |   // GL 3.2 + GLSL 150
 39 |   const char* glsl_version = "#version 150";
 40 |   glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
 41 |   glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 2);
 42 |   glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);  // 3.2+ only
 43 |   glfwWindowHint(GLFW_OPENGL_FORWARD_COMPAT, GL_TRUE);            // Required on Mac
 44 | #else
 45 |   // GL 3.0 + GLSL 130
 46 |   const char* glsl_version = "#version 130";
 47 |   glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
 48 |   glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 0);
 49 |   //glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);  // 3.2+ only
 50 |   //glfwWindowHint(GLFW_OPENGL_FORWARD_COMPAT, GL_TRUE);            // 3.0+ only
 51 | #endif
 52 | 
 53 |   // Create window with graphics context
 54 |   GLFWwindow* window = glfwCreateWindow(1280, 720, "Hello Deep Learning", NULL, NULL);
 55 |   if (window == NULL)
 56 |     return 1;
 57 |   glfwMakeContextCurrent(window);
 58 |   glfwSwapInterval(1); // Enable vsync
 59 | 
 60 |   // Setup Dear ImGui context
 61 |   IMGUI_CHECKVERSION();
 62 |   ImGui::CreateContext();
 63 |   ImPlot::CreateContext();
 64 |   ImGuiIO& io = ImGui::GetIO(); (void)io;
 65 |   //io.ConfigFlags |= ImGuiConfigFlags_NavEnableKeyboard;     // Enable Keyboard Controls
 66 |   //io.ConfigFlags |= ImGuiConfigFlags_NavEnableGamepad;      // Enable Gamepad Controls
 67 | 
 68 |   // Setup Dear ImGui style
 69 |   ImGui::StyleColorsDark();
 70 |   //    ImGui::StyleColorsLight();
 71 | 
 72 |   // Setup Platform/Renderer backends
 73 |   ImGui_ImplGlfw_InitForOpenGL(window, true);
 74 |   ImGui_ImplOpenGL3_Init(glsl_version);
 75 | 
 76 |   io.Fonts->AddFontFromFileTTF("/usr/share/fonts/truetype/msttcorefonts/Comic_Sans_MS.ttf", 18.0f);
 77 | 
 78 |   // Our state
 79 |   ImVec4 clear_color = ImVec4(0.45f, 0.55f, 0.60f, 1.00f);
 80 | 
 81 |   shared_ptr<HyperParameters> hp;
 82 |   // Main loop
 83 |   while (!glfwWindowShouldClose(window))
 84 |     {
 85 |       auto hp = *g_hyper;
 86 |       glfwPollEvents();
 87 | 
 88 |       // Start the Dear ImGui frame
 89 |       ImGui_ImplOpenGL3_NewFrame();
 90 |       ImGui_ImplGlfw_NewFrame();
 91 |       ImGui::NewFrame();
 92 | 
 93 |       //      ImPlot::ShowDemoWindow();
 94 |       {
 95 | 
 96 |         ImGui::Begin("Hello, deep learning!");                          // Create a window called "Hello, world!" and append into it.
 97 | 
 98 |         ImGui::SliderInt("Batch multiple", &hp.batchMult, 1, 32);
 99 |         ImGui::SliderFloat("Learning rate", &hp.lr, 0.0f, 0.2f);
100 |         ImGui::SliderFloat("Momentum", &hp.momentum, 0.0f, 1.0f);
101 | 
102 | 
103 |         if (ImGui::Button("Button"))                            // Buttons return true when clicked (most widgets return true when edited/activated)
104 |           std::cout<<"Clicked!"<<std::endl;
105 |         ImGui::SameLine();
106 |         ImGui::Text("Have %ld active nodes", TrackedNumberImp<float>::getCount());
107 |         ImGui::Text("Trained %d characters", g_progress.trained.load());
108 |         ImGui::Text("Last batch of %d characters took %.1f seconds", hp.getBatchSize(), g_progress.lastTook);
109 | 
110 |         if (g_progress.losses.size()>0 && ImPlot::BeginPlot("Loss plot")) {
111 |           //          ImPlot::SetupAxes(NULL,NULL,ImPlotAxisFlags_AutoFit|ImPlotAxisFlags_RangeFit, ImPlotAxisFlags_AutoFit|ImPlotAxisFlags_RangeFit);
112 |           ImPlot::SetupAxes("Batch", "Loss");
113 |           ImPlot::SetupAxisLimits(ImAxis_Y1, 2, 7);
114 |           ImPlot::SetupAxis(ImAxis_Y2, "Correct",ImPlotAxisFlags_AuxDefault);
115 |           ImPlot::SetupAxisLimits(ImAxis_Y2, 0, 100);
116 |           ImPlot::SetAxes(ImAxis_X1, ImAxis_Y1);
117 |           ImPlot::PlotLine("Loss", &g_progress.losses[0], g_progress.losses.size());
118 |           ImPlot::SetAxes(ImAxis_X1, ImAxis_Y2);
119 |           ImPlot::PlotLine("Corrects", &g_progress.corrects[0], g_progress.corrects.size());
120 |           ImPlot::EndPlot();
121 |         }
122 |         
123 |         ImGui::End();
124 |       }
125 | 
126 |         
127 |       // Rendering
128 |       ImGui::Render();
129 |       int display_w, display_h;
130 |       glfwGetFramebufferSize(window, &display_w, &display_h);
131 |       glViewport(0, 0, display_w, display_h);
132 |       glClearColor(clear_color.x * clear_color.w, clear_color.y * clear_color.w, clear_color.z * clear_color.w, clear_color.w);
133 |       glClear(GL_COLOR_BUFFER_BIT);
134 |       ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData());
135 | 
136 |       glfwSwapBuffers(window);
137 |       auto newhyper = make_shared<HyperParameters>(hp);
138 |       g_hyper.swap(newhyper);
139 |     }
140 | 
141 |   // Cleanup
142 |   ImGui_ImplOpenGL3_Shutdown();
143 |   ImGui_ImplGlfw_Shutdown();
144 |   ImPlot::DestroyContext();
145 |   ImGui::DestroyContext();
146 | 
147 | 
148 |   glfwDestroyWindow(window);
149 |   glfwTerminate();
150 |   return 0;
151 | }
152 | 


--------------------------------------------------------------------------------
/alternate/workertest.cc:
--------------------------------------------------------------------------------
  1 | #include "tracked.hh"
  2 | #include "array.hh"
  3 | #include "ext/doctest.h"
  4 | 
  5 | using namespace std;
  6 | 
  7 | TEST_CASE("basic worker test") {
  8 |   TrackedFloat a(2.0), b(3.0), c(1.0);
  9 |   auto d = a * b + c;
 10 |   a.setVariable();
 11 |   float res = d.getVal();
 12 |   cout<<"res is: "<<res<<endl;
 13 | 
 14 |   auto topo = d.getTopo();
 15 | 
 16 |   auto w = d.getWork<float>(topo);
 17 |   for(const auto& g : w.dyns) {
 18 |     cout<<"Setting variable ["<<g.first<<"] to "<<g.second->d_val<<"\n";
 19 |     w.work[g.first].ourval = g.second->d_val;
 20 |   }
 21 | 
 22 | 
 23 |   int num=0;
 24 |   for(const auto& item : w.work) {
 25 |     cout<< "["<<num<<"] "<<item <<endl;
 26 |     num++;
 27 |   }
 28 | 
 29 |   cout<<"worker res: "<<w.getResult()<<endl;
 30 |   a.impl->d_val = 1.0;
 31 |   d.backward();
 32 |   d.zeroGrad();
 33 |   
 34 |   cout<<"New value: "<<d.getVal() << endl;
 35 |   w.syncVariable();
 36 |   cout<<"New worker val after resync: "<<w.getResult()<<endl;
 37 |   CHECK(w.getResult() == d.getVal());
 38 | }
 39 | 
 40 | TEST_CASE("grad worker test") {
 41 |   cout<<"START!"<<endl;
 42 |   TrackedFloat a(2.0), b(3.0), c(1.0);
 43 |   auto d = a * a * a + b * b;
 44 |   a.setVariable();
 45 |   b.needsGrad();
 46 |   c.needsGrad();
 47 |   
 48 |   float res = d.getVal();
 49 |   d.backward();
 50 | 
 51 |   cout<<"res is: "<<res<<", a grad is: " << a.getGrad()<<endl;
 52 |   cout<<"b grad is: " << b.getGrad()<<endl;
 53 |   cout<<"c grad is: " << c.getGrad()<<endl;
 54 | 
 55 |   auto topo = d.getTopo();
 56 |   auto w = d.getWork<float>(topo);
 57 |   float wresult = w.getResult();
 58 |   cout<<"Worker result: "<<wresult<<endl;
 59 | 
 60 |   CHECK(res == wresult);  
 61 |   w.backward();
 62 |   // syncback needgrads
 63 | 
 64 |   int num=0;
 65 |   for(const auto& item : w.work) {
 66 |     cout<< "["<<num<<"] "<<item <<endl;
 67 |     num++;
 68 |   }
 69 | }
 70 | 
 71 | TEST_CASE("array worker test") {
 72 |   cout<<"Array worker test"<<endl;
 73 |   NNArray<float, 2, 2> img, weights;
 74 |   img.randomize();
 75 |   weights.randomize();
 76 |   weights.needsGrad();
 77 | 
 78 |   auto c = weights * img;
 79 |   auto sumval = c.sum();
 80 |   float res = sumval.getVal();
 81 |   sumval.backward();
 82 |   img.zeroGrad(); // backward populates grads, when it shouldn't
 83 |   cout<<"res is: "<<res<<", grad of weights(0,0): "<<weights(0,0).getGrad()<<", val of weights(0,0) "<<weights(0,0).getVal()<<endl;
 84 | 
 85 |   auto topo = sumval.getTopo();
 86 |   auto wgrad = weights.getGrad();
 87 |   
 88 |   auto w = sumval.getWork<float>(topo);
 89 |   CHECK(w.dyns.size() == 8);
 90 |   cout<<"worker res: "<<w.getResult()<<endl;
 91 | 
 92 |   CHECK(w.getResult() == sumval.getVal());
 93 |   
 94 |   cout<<"a(0,0).getGrad() "<< weights(0,0).getGrad()<<endl;
 95 |   w.backward();
 96 |   int num=0;
 97 |   for(const auto& item : w.work) {
 98 |     cout<< "["<<num<<"] "<<item << " grad " <<w.grads[num]<<endl;
 99 |     num++;
100 |   }
101 |   w.syncGrad();
102 | 
103 |   auto wgrad2 = weights.getGrad();
104 |   CHECK(wgrad2 == wgrad);
105 | 
106 |   SArray<float, 2, 2> zero;
107 |   zero.setZero();
108 |   auto imggrad = img.getGrad();
109 |   CHECK(imggrad == zero);
110 |   
111 |   cout<<"sizeof: "<< sizeof(w.work[0]) <<endl;
112 | }
113 | 
114 | 
115 | TEST_CASE("array worker projection test") {
116 | 
117 |   NNArray<float, 2, 5> a;
118 |   NNArray<float, 5, 2> b;
119 |   a.randomize();
120 |   b.randomize();
121 |   TrackedFloat res = (a * b).sum();
122 |   float nresult = res.getVal();
123 | 
124 |   auto topo = res.getTopo();
125 |   auto w=res.getWork<float>(topo);
126 | 
127 |   auto proj = makeProj(a, w);
128 |   
129 |   float wresult = w.getResult();
130 |   CHECK(nresult == wresult);
131 | 
132 |   a(0,0) = a(0,0).getVal() + 1;
133 |   res.zeroGrad(); // flushes cache
134 | 
135 |   projForward(proj, a, w);
136 |   cout<<"nresult: "<<nresult<<endl;
137 |   CHECK(res.getVal() == w.getResult());
138 | }
139 | 
140 | TEST_CASE("array worker AVX2 test") {
141 | 
142 |   NNArray<float, 2, 5> a;
143 |   NNArray<float, 5, 2> b;
144 |   a.randomize();
145 |   b.randomize();
146 |   TrackedFloat res = (a * b).sum();
147 |   float nresult = res.getVal();
148 | 
149 |   auto topo = res.getTopo();
150 |   auto w=res.getWork<fvector<8>>(topo);
151 | 
152 |   auto proj = makeProj(a, w);
153 |   
154 |   auto wresult = w.getResult();
155 |   CHECK(nresult == wresult.v[0]);
156 |   CHECK(nresult == wresult.v[1]);
157 |   CHECK(nresult == wresult.v[2]);
158 |   CHECK(nresult == wresult.v[7]);
159 | 
160 |   a(0,0) = a(0,0).getVal() + 1;
161 |   res.zeroGrad(); // flushes cache
162 | 
163 |   projForward(proj, a, w);
164 |   cout<<"nresult: "<<nresult<<endl;
165 |   CHECK(res.getVal() == w.getResult().v[3]);
166 | 
167 |   
168 | }
169 | 
170 | TEST_CASE("worker AVX2 test") {
171 | 
172 |   NNArray<float, 1, 5> a;
173 |   NNArray<float, 1, 5> b;
174 |   a.zero();
175 |   b.zero();
176 |   a.setVariable();
177 |   b.setVariable();
178 |   
179 |   auto c = a+b;
180 |   auto res = c(0,0);
181 |   float nresult = res.getVal();
182 | 
183 |   CHECK(nresult == 0.0);
184 |   
185 |   auto topo = res.getTopo();
186 | 
187 |   auto w = res.getWork<float>(topo);
188 |   auto aproj = makeProj(a, w);
189 |   auto bproj = makeProj(b, w);
190 |   
191 |   auto w8=res.getWork<fvector<8>>(topo);
192 |   
193 |   auto wresult = w8.getResult();
194 |   CHECK(wresult.v[0] == 0 );
195 |   CHECK(wresult.v[1] == 0 );
196 |   CHECK(wresult.v[2] == 0 );
197 |   CHECK(wresult.v[3] == 0 );
198 |     
199 | 
200 |   NNArray<fvector<8>, 1, 5> a8;
201 |   a8(0,0) = 0;
202 |   a8(0,0).impl->d_val.v[0]=1;
203 |   a8(0,0).impl->d_val.v[1]=2;
204 |   a8(0,0).impl->d_val.v[2]=4;
205 | 
206 | 
207 |   NNArray<fvector<8>, 1, 5> b8;
208 |   b8(0,0) = 0;
209 |   b8(0,0).impl->d_val.v[0]=4;
210 |   b8(0,0).impl->d_val.v[1]=5;
211 |   b8(0,0).impl->d_val.v[2]=6;
212 |   
213 |   
214 |   res.zeroGrad(); // flushes cache
215 | 
216 |   projForward(aproj, a8, w8);
217 |   projForward(bproj, b8, w8);
218 |   
219 |   wresult = w8.getResult();
220 |   CHECK(wresult.v[0] == 5);
221 |   CHECK(wresult.v[1] == 7);
222 |   CHECK(wresult.v[2] == 10);
223 | 
224 |   
225 | }
226 | 


--------------------------------------------------------------------------------
/gru.hh:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include "layers.hh"
  3 | 
  4 | 
  5 | // hidden state=> dense linear => output x
  6 | 
  7 | // x is input
  8 | // h_t, h_{t-1} = hidden state
  9 | 
 10 | // gate_{reset} = \sigma(W_{input_{reset}} \cdot x_t + W_{hidden_{reset}} \cdot h_{t-1})
 11 | 
 12 | // W_input_reset  - ^^ normal matrix products
 13 | // W_input_hidden
 14 | 
 15 | // pytorch:
 16 | 
 17 | // r_t​ = σ(W_{ir} ​x_t + b_{ir}​+W_{hr}​h_{t−1}​ +b_{hr}​)            // reset gate
 18 | // z_t​ = σ(W_{iz} ​x_t ​+ b_{iz} ​+W_{hz}​h_{t−1}​ + b_{hz}​)          // update
 19 | // n_t​ = tanh(W_{in}​x_t​+b_{in}​+ r_t​*(W_{hn} ​h_{t−1}​ + b_{hn}​))   // "new" - * is dotproduct
 20 | // h_t​=(1−z_t​)*n_t​+z_t​*h_{t−1)​}                                  // new h
 21 | 
 22 | // the hidden state is also the output, which needs linear combination to turn into input size again
 23 | // https://pytorch.org/docs/stable/generated/torch.nn.GRU.html
 24 | template<typename T, unsigned int IN, unsigned int HIDDEN>
 25 | struct GRULayer : LayerBase
 26 | {
 27 |   NNArray<T, HIDDEN, IN> d_w_ir; // reset
 28 |   NNArray<T, HIDDEN, IN> d_w_iz; // update
 29 |   NNArray<T, HIDDEN, IN> d_w_in; // new
 30 | 
 31 |   NNArray<T, HIDDEN, HIDDEN> d_w_hr;  // hidden reset
 32 |   NNArray<T, HIDDEN, HIDDEN> d_w_hz; // hidden update
 33 |   NNArray<T, HIDDEN, HIDDEN> d_w_hn; // hidden "new"
 34 | 
 35 |   NNArray<T, HIDDEN, 1> d_prevh;
 36 | 
 37 |   std::array<unsigned int, decltype(d_w_ir)::SIZE> d_w_ir_proj;
 38 |   std::array<unsigned int, decltype(d_w_iz)::SIZE> d_w_iz_proj;
 39 |   std::array<unsigned int, decltype(d_w_in)::SIZE> d_w_in_proj;
 40 | 
 41 |   std::array<unsigned int, decltype(d_w_hr)::SIZE> d_w_hr_proj;
 42 |   std::array<unsigned int, decltype(d_w_hz)::SIZE> d_w_hz_proj;
 43 |   std::array<unsigned int, decltype(d_w_hn)::SIZE> d_w_hn_proj;
 44 | 
 45 |   GRULayer()
 46 |   {
 47 |     randomize();
 48 |   }
 49 | 
 50 |   // https://blog.floydhub.com/gru-with-pytorch/
 51 |   //  https://towardsdatascience.com/gate-recurrent-units-explained-using-matrices-part-1-3c781469fc18
 52 |   // these appear to be slightly different
 53 |   auto forward(const NNArray<T, IN, 1>& xt)
 54 |   {
 55 |     auto r_t = (d_w_ir * xt + d_w_hr * d_prevh).applyFunc(SigmoidFunc()); // reset gate
 56 |     auto z_t = (d_w_iz * xt + d_w_hz * d_prevh).applyFunc(SigmoidFunc());
 57 |     auto n_t = (d_w_in * xt + r_t.dot(d_w_hn *d_prevh)).applyFunc(TanhFunc());
 58 |     NNArray<T, HIDDEN, 1> one;
 59 |     one.constant(1.0);
 60 |     auto h_t = (one - z_t).dot(n_t) + z_t.dot(d_prevh);
 61 |     d_prevh = h_t; // "this is where the magic happens"
 62 |     return h_t;
 63 |   }
 64 | 
 65 |   void learn(float lr) override
 66 |   {
 67 |     { auto grad1 = d_w_ir.getGrad(); grad1 *= lr; d_w_ir -= grad1;  }
 68 |     { auto grad1 = d_w_iz.getGrad(); grad1 *= lr; d_w_iz -= grad1;  }
 69 |     { auto grad1 = d_w_in.getGrad(); grad1 *= lr; d_w_in -= grad1;  }
 70 |     
 71 |     { auto grad1 = d_w_hr.getGrad(); grad1 *= lr; d_w_hr -= grad1;  }
 72 |     { auto grad1 = d_w_hz.getGrad(); grad1 *= lr; d_w_hz -= grad1;  }
 73 |     { auto grad1 = d_w_hn.getGrad(); grad1 *= lr; d_w_hn -= grad1;  }
 74 |   }
 75 | 
 76 |   void save(std::ostream& out) const override
 77 |   {
 78 |     d_w_ir.save(out);
 79 |     d_w_iz.save(out);
 80 |     d_w_in.save(out);
 81 |     
 82 |     d_w_hr.save(out);
 83 |     d_w_hz.save(out);
 84 |     d_w_hn.save(out);
 85 |   }
 86 |   void load(std::istream& in) override
 87 |   {
 88 |     d_w_ir.load(in);
 89 |     d_w_iz.load(in);
 90 |     d_w_in.load(in);
 91 |     
 92 |     d_w_hr.load(in);
 93 |     d_w_hz.load(in);
 94 |     d_w_hn.load(in);
 95 | 
 96 |   }
 97 |   void randomize() // "Xavier initialization"  http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf
 98 |   {
 99 |     d_w_ir.randomize(1.0/sqrt(HIDDEN));
100 |     d_w_iz.randomize(1.0/sqrt(HIDDEN));
101 |     d_w_in.randomize(1.0/sqrt(HIDDEN));
102 | 
103 |     d_w_hr.randomize(1.0/sqrt(HIDDEN));
104 |     d_w_hz.randomize(1.0/sqrt(HIDDEN));
105 |     d_w_hn.randomize(1.0/sqrt(HIDDEN));
106 |     d_prevh.zero();
107 | 
108 |     d_w_ir.needsGrad();
109 |     d_w_iz.needsGrad();
110 |     d_w_in.needsGrad();
111 |     
112 |     d_w_hr.needsGrad();
113 |     d_w_hz.needsGrad();
114 |     d_w_hn.needsGrad();
115 |   }
116 | 
117 |   unsigned int size() const override
118 |   {
119 |     return d_w_ir.size() + d_w_iz.size() + d_w_in.size() +
120 |            d_w_hr.size() + d_w_hz.size() + d_w_hn.size();
121 |   }
122 |   
123 |   void zeroGrad() override
124 |   {
125 |     d_w_ir.zeroGrad();
126 |     d_w_iz.zeroGrad();
127 |     d_w_in.zeroGrad();
128 | 
129 |     d_w_hr.zeroGrad();
130 |     d_w_hz.zeroGrad();
131 |     d_w_hn.zeroGrad();
132 |   }
133 | 
134 |   void addGrad(const GRULayer& rhs)
135 |   {
136 |     d_w_ir.addGrad(rhs.d_w_ir.getGrad());
137 |     d_w_iz.addGrad(rhs.d_w_iz.getGrad());
138 |     d_w_in.addGrad(rhs.d_w_in.getGrad());
139 | 
140 |     d_w_hr.addGrad(rhs.d_w_hr.getGrad());
141 |     d_w_hz.addGrad(rhs.d_w_hz.getGrad());
142 |     d_w_hn.addGrad(rhs.d_w_hn.getGrad());
143 |     
144 |   }
145 | 
146 |   void setGrad(const GRULayer& rhs, float divisor)
147 |   {
148 |     d_w_ir.setGrad(rhs.d_w_ir.getGrad()/divisor);
149 |     d_w_iz.setGrad(rhs.d_w_iz.getGrad()/divisor);
150 |     d_w_in.setGrad(rhs.d_w_in.getGrad()/divisor);
151 | 
152 |     d_w_hr.setGrad(rhs.d_w_hr.getGrad()/divisor);
153 |     d_w_hz.setGrad(rhs.d_w_hz.getGrad()/divisor);
154 |     d_w_hn.setGrad(rhs.d_w_hn.getGrad()/divisor);
155 |   }
156 | 
157 |   template<typename W>
158 |   void makeProj(const W& w)
159 |   {
160 |     d_w_ir_proj = ::makeProj(d_w_ir, w);
161 |     d_w_iz_proj = ::makeProj(d_w_iz, w);
162 |     d_w_in_proj = ::makeProj(d_w_in, w);
163 | 
164 |     d_w_hr_proj = ::makeProj(d_w_hr, w);
165 |     d_w_hz_proj = ::makeProj(d_w_hz, w);
166 |     d_w_hn_proj = ::makeProj(d_w_hn, w);
167 | 
168 |   }
169 |   template<typename W>
170 |   void projForward(W& w) const
171 |   {
172 |     ::projForward(d_w_ir_proj, d_w_ir, w);
173 |     ::projForward(d_w_iz_proj, d_w_iz, w);
174 |     ::projForward(d_w_in_proj, d_w_in, w);
175 | 
176 |     ::projForward(d_w_hr_proj, d_w_hr, w);
177 |     ::projForward(d_w_hz_proj, d_w_hz, w);
178 |     ::projForward(d_w_hn_proj, d_w_hn, w);
179 |     
180 |   }
181 | 
182 |   template<typename W>
183 |   void projBackGrad(const W& w)
184 |   {
185 |     ::projBackGrad(d_w_ir_proj, w, d_w_ir);
186 |     ::projBackGrad(d_w_iz_proj, w, d_w_iz);
187 |     ::projBackGrad(d_w_in_proj, w, d_w_in);
188 | 
189 |     ::projBackGrad(d_w_hr_proj, w, d_w_hr);
190 |     ::projBackGrad(d_w_hz_proj, w, d_w_hz);
191 |     ::projBackGrad(d_w_hn_proj, w, d_w_hn);
192 |   }
193 | 
194 |   
195 | };
196 | 
197 | 


--------------------------------------------------------------------------------
/los.cc:
--------------------------------------------------------------------------------
  1 | 
  2 | #include <malloc.h>
  3 | #include <fenv.h>
  4 | #include <random>
  5 | #include <chrono>
  6 | #include <fstream>
  7 | #include "fvector.hh"
  8 | #include <iostream>
  9 | #include <array>
 10 | #include "gru.hh"
 11 | #include "textsupport.hh"
 12 | 
 13 | std::ofstream g_tree;//("./tree.part");
 14 | #include "tracked.hh"
 15 | #include "misc.hh"
 16 | 
 17 | #include <initializer_list>
 18 | using namespace std;
 19 | 
 20 | template<typename T>
 21 | struct GRUModel
 22 | {
 23 |   struct State
 24 |   {
 25 |     GRULayer<T, 98, 250> gm1;
 26 |     GRULayer<T, 250, 250> gm2;
 27 |     Linear<T, 250, 98> fc;
 28 |     void zeroGrad()
 29 |     {
 30 |       gm1.zeroGrad();
 31 |       gm2.zeroGrad();
 32 |       fc.zeroGrad();
 33 |     }
 34 |     
 35 |     void save(std::ostream& out) const
 36 |     {
 37 |       gm1.save(out);
 38 |       gm2.save(out);
 39 |       fc.save(out);
 40 |     }
 41 | 
 42 |     void load(std::istream& in) 
 43 |     {
 44 |       gm1.load(in);
 45 |       gm2.load(in);
 46 |       fc.load(in);
 47 |     }
 48 | 
 49 |     template<typename W>
 50 |     void makeProj(const W& w)
 51 |     {
 52 |       gm1.makeProj(w);
 53 |       gm2.makeProj(w);
 54 |       fc.makeProj(w);
 55 |     }
 56 | 
 57 |     template<typename W>
 58 |     void projForward(W& w) const
 59 |     {
 60 |       gm1.projForward(w);
 61 |       gm2.projForward(w);
 62 |       fc.projForward(w);
 63 |     }
 64 |     template<typename W>
 65 |     void projBackGrad(const W& w) 
 66 |     {
 67 |       gm1.projBackGrad(w);
 68 |       gm2.projBackGrad(w);
 69 |       fc.projBackGrad(w);
 70 |     }
 71 |   };
 72 |   vector<NNArray<T, 98, 1>> invec;
 73 |   vector<NNArray<T, 1, 98>> expvec;
 74 |   vector<NNArray<T, 98, 1>> scorevec;
 75 | 
 76 |   TrackedNumber<T> totloss;
 77 |   
 78 |   void unroll(State& s, unsigned int choplen)
 79 |   {
 80 |     cout<<"Unrolling the GRU";
 81 |     totloss = TrackedNumber<T>(0.0);
 82 |     for(size_t i = 0 ; i < choplen; ++i) {
 83 |       cout<<"."; cout.flush();
 84 |       NNArray<T, 98, 1> in;
 85 |       NNArray<T, 1, 98> expected;
 86 |       in.zero();  
 87 |       expected.zero(); 
 88 |       
 89 |       invec.push_back(in);
 90 |       expvec.push_back(expected);
 91 |       auto res1 = s.fc.forward(s.gm2.forward(s.gm1.forward(in)));
 92 |       auto score = res1.logSoftMax();
 93 |       scorevec.push_back(score);
 94 |       auto loss = TrackedNumber<T>(0.0) - (expected*score)(0,0);
 95 |       totloss = totloss + loss;
 96 |     }
 97 |     totloss = totloss/TrackedNumber<T>(choplen);
 98 |     cout<<"\n";
 99 |   }
100 | 
101 | };
102 | 
103 | 
104 | int main(int argc, char **argv)
105 | {
106 |   BiMapper bm("corpus.txt", 98);
107 |   constexpr int choplen= 75;
108 |   vector<string> sentences=textChopper("corpus.txt", choplen, 10);
109 |   cout<<"Got "<<sentences.size()<<" sentences"<<endl;
110 |   Batcher batcher(sentences.size());
111 | 
112 |   auto grum = make_unique<GRUModel<float>>();
113 |   GRUModel<float>::State s;
114 | 
115 |   if(argc > 1) {
116 |     cout<<"Loading model state from "<<argv[1]<<endl;
117 |     loadModelState(s, argv[1]);
118 |   }
119 |   grum->unroll(s, choplen - 1);
120 | 
121 |   constexpr int batchsize = 64;
122 | 
123 |   cout<<"\nDoing topo.."; cout.flush();
124 |   auto topo = grum->totloss.getTopo();
125 |   cout<<" "<< topo.size() <<" entries"<<endl;
126 | 
127 |   vector<std::array<unsigned int, 98>> invecprojarray, expvecprojarray, scorevecprojarray;
128 |   cout<<"Making float worker"<<endl;
129 |   for(auto& x : grum->scorevec) {
130 |     x.setVariable();
131 |   }
132 |   for(auto& x : grum->invec) {
133 |     x.setVariable();
134 |   }
135 |   for(auto& x : grum->expvec) {
136 |     x.setVariable();
137 |   }
138 | 
139 |   auto w = grum->totloss.getWork<float>(topo);
140 |   
141 |   for(const auto& x : grum->scorevec) {
142 |     scorevecprojarray.push_back(makeProj(x, w));
143 |   }
144 |   
145 |   for(const auto& x : grum->invec)
146 |     invecprojarray.push_back(makeProj(x, w));
147 |   
148 |   for(const auto& x : grum->expvec)
149 |     expvecprojarray.push_back(makeProj(x, w));
150 |   
151 |   s.makeProj(w);
152 |   
153 |   cout<<"Resetting GRU"<<endl;
154 |   cout<<TrackedNumberImp<float>::getCount()<<" instances before clean"<<endl;
155 |     
156 |   grum.reset();
157 |   cout<<TrackedNumberImp<float>::getCount()<<" instances after clean"<<endl;
158 |     
159 |   vector<NNArray<fvector<8>, 98, 1>> invec(choplen);
160 |   vector<NNArray<fvector<8>, 1, 98>> expvec(choplen);
161 |   vector<NNArray<fvector<8>, 98, 1>> scorevec(choplen);
162 | 
163 |   cout<<"Making AVX2 worker"<<endl;
164 |   auto w8 = w.convert<fvector<8>>(); 
165 | 
166 |   cout<<"Starting the work"<<endl;
167 | 
168 |   for(;;) { // the batch loop
169 |     cout<<TrackedNumberImp<float>::getCount()<<" instances"<<endl;
170 |     double batchloss = 0;
171 |     w8.zeroGrad();
172 |     s.zeroGrad();
173 |     s.projForward(w8);
174 |     for(unsigned int minibatchno = 0 ; minibatchno < batchsize/8; ++minibatchno) {
175 |       s.gm1.d_prevh.reset();
176 |       s.gm2.d_prevh.reset();
177 | 
178 |       auto minibatch = batcher.getBatch(8);
179 |       if(minibatch.size() != 8)
180 |         goto batcherEmpty;
181 | 
182 |       for(size_t pos = 0 ; pos < choplen - 1; ++pos) {
183 |         invec[pos].zero();
184 |         expvec[pos].zero();
185 |       }
186 | 
187 |       int avxindex =0;
188 |       for(const auto& idx : minibatch) {
189 |         string input = sentences[idx];
190 |         std::string output;
191 | 
192 |         for(size_t pos = 0 ; pos < input.size() - 1; ++pos) {
193 |           cout<<input.at(pos);
194 |           invec[pos](bm.c2i(input.at(pos)), 0).impl->d_val.v[avxindex] = 1.0;
195 |           expvec[pos](0, bm.c2i(input.at(pos+1))).impl->d_val.v[avxindex] = 1.0;
196 |         }
197 |         cout<<endl;
198 |         avxindex++;
199 |       }
200 | 
201 |       for(unsigned int projpos = 0; projpos < scorevecprojarray.size(); ++projpos) {
202 |         projForward(invecprojarray[projpos], invec[projpos], w8);
203 |         projForward(expvecprojarray[projpos], expvec[projpos], w8);
204 |       }
205 | 
206 |       
207 |       auto numloss = w8.getResult().sum()/8; // this triggers the whole calculation
208 | 
209 |       for(unsigned int projpos = 0; projpos < scorevecprojarray.size(); ++projpos)
210 |         projBack(scorevecprojarray[projpos], w8, scorevec[projpos]);
211 | 
212 |       
213 |       for(int t =0 ; t< 8; ++t) {
214 |         for(size_t pos = 0 ; pos < choplen - 1; ++pos) {
215 |           cout<< bm.i2c(scorevec[pos].getUnparallel(t).maxValueIndexOfColumn(0));
216 |         }
217 |         cout<<"\n";
218 |       }
219 |       batchloss += numloss;
220 |       cout<<"\naverage loss: "<<numloss<<endl;
221 |       s.projBackGrad(w8);
222 |     }
223 |     // done with all the minibatches, have a batch
224 |     
225 | 
226 | 
227 |     batchloss /= batchsize;
228 |     
229 |     float lr=0.01/batchsize;
230 |     cout<<"Average batch loss: "<<batchloss<<endl;
231 |     s.gm1.learn(lr);
232 |     s.gm2.learn(lr);
233 |     s.fc.learn(lr);
234 |     //    cout<<"\n\n";
235 |     saveModelState(s, "los-worker.state");
236 |   }
237 |  batcherEmpty:;
238 | }
239 | 
240 | 


--------------------------------------------------------------------------------
/alternate/modular-threaded-convo.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <vector>
  3 | #include <fstream>
  4 | #include <memory>
  5 | #include <optional>
  6 | #include "array.hh"
  7 | #include "tracked.hh"
  8 | #include "mnistreader.hh"
  9 | #include "misc.hh"
 10 | #include <string.h>
 11 | #include <sstream>
 12 | #include <fenv.h>
 13 | #include "cnn-alphabet.hh"
 14 | #include "fvector.hh"
 15 | #include <mutex>
 16 | #include <atomic>
 17 | #include <thread>
 18 | 
 19 | using TheModel = CNNAlphabetModel<fvector<8>>;
 20 | 
 21 | using namespace std;
 22 | 
 23 | ofstream g_tree; //("tree.part");
 24 | 
 25 | template<typename M, typename S>
 26 | void scoreModel(S& s, const MNISTReader& mntest, int batchno)
 27 | {
 28 |   unsigned int corrects=0, wrongs=0;
 29 |   static ofstream vcsv("validation3.csv");
 30 |   static bool notfirst;
 31 |   if(!notfirst) {
 32 |     vcsv<<"batchno,corperc,avgloss\n";
 33 |     notfirst=true;
 34 |   }
 35 |   
 36 |   M mod;
 37 |   mod.init(s);
 38 | 
 39 |   auto topo = mod.loss.getTopo();
 40 |   Batcher batcher(mntest.num());
 41 |   auto batch = batcher.getBatch(80);
 42 |   
 43 |   fvector<8> totLoss=0.0; 
 44 |   
 45 |   while(!batch.empty()) {
 46 |     mod.expected.zero();
 47 |     fvector<8> label;
 48 |     for(int n = 0; n < 8 ; ++n) {
 49 |       auto idx = batch.front();
 50 |       batch.pop_front();
 51 |       
 52 |       mntest.pushImage(idx, mod.img, n);
 53 |       label.v[n] = mntest.getLabel(idx) - 1;  // a == 1..
 54 |       
 55 |       mod.expected(0, label.v[n]).impl->d_val.v[n] = 1; // "one hot vector"
 56 |     }
 57 |     
 58 |     totLoss += mod.loss.getVal();
 59 |     
 60 |     for(int n = 0; n < 8; ++n) {
 61 |       int verdict = mod.scores.getUnparallel(n).maxValueIndexOfColumn(0);
 62 |       if(verdict == label.v[n])
 63 |         corrects++;
 64 |       else
 65 |         wrongs++;
 66 |     }
 67 |     mod.loss.zeroGrad(topo);
 68 |   }
 69 |   double avgLoss = totLoss.sum()/80.0;
 70 |   double perc = 100.0*corrects/(corrects+wrongs);
 71 |   cout<<perc<<"% correct, average loss "<<avgLoss<<"\n";
 72 |   vcsv << batchno <<"," << perc << ", " << avgLoss << endl; 
 73 | }
 74 | 
 75 | int main(int argc, char** argv)
 76 | {
 77 |   feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
 78 | 
 79 |   string kind = "letters"; // or digits
 80 |   MNISTReader mn("gzip/emnist-"+kind+"-train-images-idx3-ubyte.gz", "gzip/emnist-"+kind+"-train-labels-idx1-ubyte.gz");
 81 | 
 82 |   MNISTReader mntest("gzip/emnist-"+kind+"-test-images-idx3-ubyte.gz", "gzip/emnist-"+kind+"-test-labels-idx1-ubyte.gz");
 83 | 
 84 |   cout<<"Have "<<mn.num()<<" images for training, "<<mntest.num()<<" for validation"<<endl;
 85 |   
 86 |   TheModel::State s;
 87 | 
 88 |   if(argc > 1) {
 89 |     cout<<"Loading model state from "<<argv[1]<<endl;
 90 |     loadModelState(s, argv[1]);
 91 |   }
 92 | 
 93 |   TheModel mod;
 94 |   mod.init(s);
 95 | 
 96 |   cout<<"Getting topo"<<endl;
 97 |   auto topo = mod.loss.getTopo();
 98 | 
 99 |   struct ModelEnv
100 |   {
101 |     TheModel mod;
102 |     TheModel::State s;
103 |     vector<TrackedNumberImp<fvector<8>>*> topo; // UGH
104 |   };
105 |   vector<ModelEnv> threadmods(4);
106 |   for(auto& tm : threadmods) {
107 |     cout<<"Building thread..\n";
108 |     tm.mod.init(tm.s);
109 |     tm.topo = tm.mod.loss.getTopo();
110 |   }
111 |   
112 |   cout<<TrackedNumberImp<float>::getCount()<<" instances, model has "<<s.size()<<" parameters, "<<topo.size()<<" nodes to be visited\n";
113 | 
114 |   unsigned int batchno=0;
115 |   ofstream tcsv("training3.csv");
116 |   tcsv<<"batchno,corperc,avgloss"<<endl;
117 | 
118 |   constexpr int batchsize=64;
119 |   std::mutex lock;  
120 |   
121 |   for(;;) {
122 |     Batcher batcher(mn.num()); // badgerbadger
123 | 
124 |     for(int tries=0;;++tries) {
125 |       if(!((tries+0) %32))
126 |         scoreModel<TheModel, TheModel::State>(s, mntest, batchno);
127 |       DTime dt;
128 |       dt.start();
129 | 
130 |       auto batch = batcher.getBatch(batchsize);
131 |       if(batch.size() != batchsize)
132 |         break;
133 | 
134 |       fvector<8> totLoss=0.0; 
135 |       std::atomic<unsigned int> corrects=0, wrongs=0;
136 | 
137 |       TheModel::State gather;
138 |       gather.zeroGrad();
139 |       
140 |       while(!batch.empty()) {
141 |         string modstate;
142 |         s.save(modstate);
143 |         for(auto& tm : threadmods) 
144 |           tm.s.load(modstate);
145 |         
146 |         auto work = [&batch, &mn, &totLoss, &gather, &corrects, &wrongs, &lock](ModelEnv* me) {
147 |           for(int loops = 0 ; loops < 2; ++loops) {
148 |             me->mod.expected.zero();
149 |             fvector<8> label = 0;
150 |             for(int n = 0; n < 8 ; ++n) {
151 |               unsigned int idx;
152 |               {
153 |                 std::lock_guard<std::mutex> m(lock);
154 |                 
155 |                 idx = batch.front(); // XXX MUTEX
156 |                 batch.pop_front();  // XXX MUTEX
157 |               }
158 |               mn.pushImage(idx, me->mod.img, n);
159 |               label.v[n] = mn.getLabel(idx) - 1;  // v == 1..
160 |               
161 |               me->mod.expected(0, label.v[n]).impl->d_val.v[n] = 1; // "one hot vector"
162 |             }
163 |             
164 |             auto loss = me->mod.loss.getVal();
165 |             
166 |             me->mod.loss.backward(me->topo);
167 |             {
168 |               std::lock_guard<std::mutex> m(lock);
169 |               totLoss += loss; // XXX MUTEX
170 |               gather.addGrad(me->s); // XXX MUTEX
171 |             }
172 |             
173 |             for(int n = 0; n < 8; ++n) {
174 |               int verdict = me->mod.scores.getUnparallel(n).maxValueIndexOfColumn(0);
175 |               
176 |               if(corrects + wrongs == 0) {
177 |                 cout<<"Predicted: '"<< (char)('a'+verdict)<<"', actual: '"<< (char)('a'+label.v[n]) <<"': ";
178 |                 if(verdict == label.v[n])
179 |                   cout<<"We got it right!"<<endl;
180 |                 else
181 |                   cout<<"More learning to do.."<<endl;
182 |                 cout<<"Loss: "<<me->mod.loss.getVal().v[n]<<", "<<me->mod.scores.getUnparallel(n).flatViewCol()<<endl;
183 |                 printImg(me->mod.img.getUnparallel(n));
184 |               }
185 |               
186 |               if(verdict == label.v[n])
187 |                 corrects++;
188 |               else
189 |                 wrongs++;
190 |             }
191 |             
192 |             me->mod.loss.zeroGrad(me->topo);
193 |           }
194 |         };
195 |         vector<std::thread> workers;
196 | 
197 |         for(auto &tm : threadmods) {
198 |           workers.emplace_back(work, &tm);
199 |         }
200 |         for(auto &w : workers)
201 |           w.join();
202 |           
203 |       }
204 | 
205 |       double perc = 100.0*corrects/(corrects+wrongs);
206 |       cout<<"Percent batch correct: "<<perc<<"%, average loss: "<<totLoss.sum()/batchsize <<endl;
207 | 
208 |       tcsv << batchno <<"," << perc << ", " << totLoss.sum() / batchsize << endl; 
209 |       
210 |       double lr=0.005; // mnist.cpp has 0.01, plus "momentum" of 0.5, which we don't have
211 |       s.setGrad(gather, batchsize);
212 |       s.learn(lr);
213 |       batchno++;
214 |       cout<<dt.lapUsec()/1000000.0<<" seconds/batch of " << batchsize << endl;;
215 |       saveModelState(s, "cnn-lin-threaded.state");
216 |     }
217 |   }
218 | }
219 | 


--------------------------------------------------------------------------------
/los2.cc:
--------------------------------------------------------------------------------
  1 | #include <typeinfo>
  2 | #include <typeindex>
  3 | #include <malloc.h>
  4 | #include <fenv.h>
  5 | #include <random>
  6 | #include <chrono>
  7 | #include <fstream>
  8 | #include "fvector.hh"
  9 | #include <iostream>
 10 | #include <array>
 11 | #include "gru.hh"
 12 | #include "textsupport.hh"
 13 | #include <sstream>
 14 | #include <dlfcn.h>
 15 | #include <unordered_map>
 16 | #include "cnn-alphabet.hh"
 17 | 
 18 | std::ofstream g_tree;//("./tree.part");
 19 | #include "tracked.hh"
 20 | #include "misc.hh"
 21 | #include <fstream>
 22 | 
 23 | #include <initializer_list>
 24 | using namespace std;
 25 | 
 26 | template<typename T>
 27 | struct GRUModel
 28 | {
 29 |   struct State
 30 |   {
 31 |     GRULayer<T, 98, 250> gm1;
 32 |     GRULayer<T, 250, 250> gm2;
 33 |     Linear<T, 250, 98> fc;
 34 |     void zeroGrad()
 35 |     {
 36 |       gm1.zeroGrad();
 37 |       gm2.zeroGrad();
 38 |       fc.zeroGrad();
 39 |     }
 40 |     void addGrad(const State& rhs)
 41 |     {
 42 |       gm1.addGrad(rhs.gm1);
 43 |       gm2.addGrad(rhs.gm2);
 44 |       fc.addGrad(rhs.fc);
 45 |     }
 46 | 
 47 |     void setGrad(const State& rhs, float divisor)
 48 |     {
 49 |       gm1.setGrad(rhs.gm1, divisor);
 50 |       gm2.setGrad(rhs.gm2, divisor);
 51 |       fc.setGrad(rhs.fc, divisor);
 52 |     }
 53 |     
 54 |     void save(std::ostream& out) const
 55 |     {
 56 |       gm1.save(out);
 57 |       gm2.save(out);
 58 |       fc.save(out);
 59 |     }
 60 | 
 61 |     void load(std::istream& in) 
 62 |     {
 63 |       gm1.load(in);
 64 |       gm2.load(in);
 65 |       fc.load(in);
 66 |     }
 67 |   };
 68 |   vector<NNArray<T, 98, 1>> invec;
 69 |   vector<NNArray<T, 1, 98>> expvec;
 70 |   vector<NNArray<T, 98, 1>> scorevec;
 71 | 
 72 |   TrackedNumber<T> totloss;
 73 |   
 74 |   void unroll(State& s, unsigned int choplen)
 75 |   {
 76 |     cout<<"Unrolling the GRU";
 77 |     totloss = TrackedNumber<T>(0.0);
 78 |     for(size_t i = 0 ; i < choplen; ++i) {
 79 |       cout<<"."; cout.flush();
 80 |       NNArray<T, 98, 1> in;
 81 |       NNArray<T, 1, 98> expected;
 82 |       in.zero();  
 83 |       expected.zero(); 
 84 |       
 85 |       invec.push_back(in);
 86 |       expvec.push_back(expected);
 87 |       auto res1 = s.fc.forward(s.gm2.forward(s.gm1.forward(in)));
 88 |       auto score = res1.logSoftMax();
 89 |       scorevec.push_back(score);
 90 |       auto loss = TrackedNumber<T>(0.0) - (expected*score)(0,0);
 91 |       totloss = totloss + loss;
 92 |     }
 93 |     totloss = totloss/TrackedNumber<T>(choplen);
 94 |     cout<<"\n";
 95 |   }
 96 | 
 97 | };
 98 | 
 99 | void benchexp()
100 | {
101 |   vector<float> in;
102 |   in.resize(1000000);
103 |   std::random_device rd{};
104 |   std::mt19937 gen{rd()};
105 |   std::normal_distribution<> d{0, 1};
106 | 
107 |   for(auto& item : in) 
108 |     item = (float)d(gen);
109 | 
110 | }
111 | 
112 | int main()
113 | {
114 | 
115 |   cout << getFunc<float>(g_fss[0])(2.5) << endl;
116 |   fvector<8> x=4.2;
117 |   cout << getFunc<fvector<8>>(g_fss[0])(x) << endl;
118 |   cout << getFunc<fvector<8>>(g_fss[1])(x) << endl;
119 |   cout << getDeriv<fvector<8>>(g_fss[1])(x) << endl;
120 | 
121 |   
122 |   benchexp();
123 |   return 0;
124 |   
125 |   typedef float T;
126 |   //typedef fvector<4> T;
127 | 
128 |   auto f = std::make_unique<GRUModel<T>>();
129 |   GRUModel<T>::State s;
130 | 
131 |   unsigned int choplen = 10;
132 |   f->unroll(s, choplen - 1);
133 |   
134 |   //  f->scorevec.setVariable();
135 |   //  f->expected.setVariable();
136 |   cout<<"Getting topo"<<endl;
137 |   auto topo = f->totloss.getTopo();
138 | 
139 |   DTime dt;
140 |   dt.start();
141 |   auto val = f->totloss.getVal();
142 |   cout<<val<<", "<< dt.lapUsec()<<" usec"<<endl;
143 | 
144 |   cout<<"Conventional backward: "<<endl;
145 |   dt.start();
146 |   f->totloss.backward(topo);
147 |   cout<<dt.lapUsec()<<endl;
148 |   
149 |   cout<<"Getting work"<<endl;
150 |   auto w = f->totloss.getWork<float>(topo);
151 | 
152 |   cout<<"Dropping full model"<<endl;
153 |   f.reset();
154 |   
155 |   unsigned int funcs=0;
156 |   unordered_map<string, unsigned int> funccounts;
157 |   for(const auto& item : w.work) {
158 |     if(item.mode == TrackedNumberImp<float>::Modes::Func) {
159 |       //funccounts[w.fundevs[item.findex].name]++;
160 |       ++funcs;
161 |     }
162 |   }
163 |   cout<<"funcs "<<funcs<<endl;
164 |   for(const auto& fu : funccounts)
165 |     cout<<fu.first<<": "<<fu.second<<endl;
166 |   w.syncVariable();
167 | 
168 |   dt.start();
169 |   for(int n = 0 ; n < 10; ++n)
170 |     cout << w.getResult() << "\n";
171 |   cout<<dt.lapUsec()/10.0<<endl;
172 |   cout<<"Work items: "<<w.work.size()<<", sizeof: "<<sizeof(w.work[0])<<", memory: "<<
173 |     w.work.size()*sizeof(w.work[0])/1000000.0<<"MB"<<endl;
174 | 
175 |   
176 |   cout<<"Worker backward: ";
177 | 
178 |   dt.start();
179 |   unsigned int zerousec=0;
180 |   for(int n=0; n<10;++n) {
181 |     w.backward();
182 |     DTime dt2;
183 |     dt2.start();
184 |     w.zeroGrad();
185 |     zerousec += dt2.lapUsec();
186 |   }
187 |   cout<<(dt.lapUsec()-zerousec)/10.0<<endl;
188 | 
189 | 
190 | }
191 | 
192 | 
193 |   #if 0
194 |   {
195 |     ofstream ofs("tmp.cc");
196 |     f.loss.getCode(ofs, topo, g_dyns);
197 |   }
198 |   cout<<"dyns size: "<<g_dyns.size()<<endl;
199 |   sort(g_dyns.begin(), g_dyns.end());
200 |   //  system("clang++ -std=c++17 -O0 tmp.cc -o tmp.so -fPIC -shared");
201 |   void* handle=dlopen("./tmp.so", RTLD_NOW|RTLD_GLOBAL);
202 | 
203 |   void * fsym = dlsym(handle, "_Z7getLossv");
204 |   typedef void(*setval_t)(unsigned int, float);
205 |   setval_t setval= (setval_t)dlsym(handle, "_Z6setvaljf");
206 |   
207 |   cout<<"dyns size: "<<g_dyns.size()<<", (void*)fsym "<<(void*)fsym<<", setval "<<(void*)setval<<endl;
208 | 
209 |   for(const auto& g : g_dyns)
210 |     setval(g.first, g.second->d_val);
211 | 
212 |   typedef float(*func_t)();
213 |   func_t theFunc = (func_t)fsym;
214 |   
215 |   dt.start();
216 |   for(int n=0; n < 100; ++n)
217 |     val = theFunc();
218 |   
219 |   cout<<dt.lapUsec() /100.0<<", "<<calls/100<<endl;
220 |   cout<<"val: "<<val<<endl;
221 | 
222 |   g_dyns.clear();
223 |   for(const auto& g : g_dyns)
224 |     setval(g.first, g.second->d_val);
225 |   #endif
226 | 
227 | 
228 | #if 0
229 |   NNArray<float, 50, 10> ax;
230 |   NNArray<float, 10, 50> bx;
231 | 
232 |   ax.randomize();
233 |   bx.randomize();
234 |   auto ares = ax*bx;
235 | 
236 |   auto sum = ares.sum();
237 |   DTime dt;
238 |   dt.start();
239 |   float res = sum.getVal();
240 |   cout<<dt.lapUsec();
241 |   cout<< "// "<< res << endl;
242 | 
243 |   auto topo = sum.getTopo();
244 |   if(0)
245 |   {
246 |     ofstream ofs("tmp.cc");
247 |     sum.getCode(ofs, topo, g_dyns);
248 |   }
249 |   sort(g_dyns.begin(), g_dyns.end());
250 |   system("clang++ -std=c++17 tmp.cc -o tmp.so -fPIC -shared");
251 |   void* handle=dlopen("./tmp.so", RTLD_NOW|RTLD_GLOBAL);
252 | 
253 |   void * f = dlsym(handle, "_Z7getLossv");
254 |   
255 |   cout<<"dyns size: "<<g_dyns.size()<<", (void*)f "<<(void*)f<<endl;
256 |   typedef float(*func_t)();
257 |   func_t theFunc = (func_t)f;
258 |   dt.start();
259 |   res = theFunc();
260 |   cout<<dt.lapUsec()<<"\n";
261 |   cout<<"res: "<<res<<endl;
262 | #endif
263 | #if 0 
264 | float getval(unsigned int v)
265 | {
266 |   calls++;
267 |   return g_vect[v];
268 |   pair<unsigned int, TrackedNumberImp<float>*> fnd(v, 0);
269 |   auto iter = lower_bound(g_dyns.begin(), g_dyns.end(), fnd);
270 |   return (iter)->second->d_val;
271 | }
272 | #endif 
273 | float getLoss() __attribute__((weak));
274 | 
275 | float getLoss()
276 | {
277 |   cout<<"Weak version called"<<endl;
278 |   return -1;
279 | }
280 | 


--------------------------------------------------------------------------------
/testrunner.cc:
--------------------------------------------------------------------------------
  1 | #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
  2 | #include "ext/doctest.h"
  3 | #include "tracked.hh"
  4 | using namespace std;
  5 | 
  6 | ofstream g_tree; 
  7 | 
  8 | TEST_CASE("basic test") {
  9 |   TrackedFloat x(2.0);
 10 |   TrackedFloat y(3.0);
 11 |   TrackedFloat res = x * y;
 12 |   CHECK(res.getVal() == 6.0);
 13 | }
 14 | 
 15 | 
 16 | TEST_CASE("quadratic test") {
 17 |   TrackedFloat x(2.0);
 18 |   TrackedFloat y(3.0);
 19 |   TrackedFloat res = x + y * y; 
 20 |   CHECK(res.getVal() == 11.0);
 21 |   res.backward();
 22 |   CHECK(x.getGrad() == 1.0); 
 23 |   CHECK(y.getGrad() == 6.0); // 2 * y 
 24 | }
 25 | 
 26 | TEST_CASE("quadratic test negative") {
 27 |   TrackedFloat x(-1.0);
 28 |   TrackedFloat y(-3.0);
 29 |   TrackedFloat res = x + y * y;
 30 |   CHECK(res.getVal() == 8.0);
 31 |   res.backward();
 32 |   CHECK(x.getGrad() == 1.0);
 33 |   CHECK(y.getGrad() == -6.0);
 34 | }
 35 | 
 36 | TEST_CASE("cube test negative") {
 37 |   TrackedFloat x(-1.0);
 38 |   TrackedFloat y(-4.0);
 39 |   TrackedFloat res = x*x + y * y * y;
 40 |   CHECK(res.getVal() == -63.0);
 41 |   res.backward();
 42 |   CHECK(x.getGrad() == -2.0);
 43 |   CHECK(y.getGrad() == 48.0); // -> 3*y^2 = 48
 44 | }
 45 | 
 46 | 
 47 | TEST_CASE("addition test") {
 48 |   TrackedFloat x(-1.0);
 49 |   TrackedFloat y(-4.0);
 50 |   TrackedFloat res = x+x+x+x +y +y +y +x +x; 
 51 |   CHECK(res.getVal() == -18.0);
 52 |   res.backward();
 53 |   CHECK(x.getGrad() == 6);
 54 |   CHECK(y.getGrad() == 3);
 55 | }
 56 | 
 57 | 
 58 | TEST_CASE("subtraction test") {
 59 |   TrackedFloat x(-1.0);
 60 |   TrackedFloat y(-4.0);
 61 |   TrackedFloat res = x-x - y;
 62 |   CHECK(res.getVal() == 4.0);
 63 |   res.backward();
 64 |   CHECK(x.getGrad() == 0);
 65 |   CHECK(y.getGrad() == -1);
 66 | }
 67 | 
 68 | 
 69 | TEST_CASE("division test") {
 70 |   TrackedFloat x(12.0);
 71 |   TrackedFloat y(2.0);
 72 |   TrackedFloat res = x/y;
 73 |   CHECK(res.getVal() == 6.0);
 74 |   res.backward();     // x * y^-1 
 75 |   CHECK(x.getGrad() == 0.5); // y^-1 -> 0.5
 76 |   CHECK(y.getGrad() == -3); // -y^-2 * x = -0.25 * 12 = -3
 77 |   res.zeroGrad();
 78 | 
 79 |   res = (x+y)/y;            // x/y + 1
 80 |   CHECK(res.getVal() == 7.0);
 81 |   res.backward();
 82 |   CHECK(x.getGrad() == 0.5);
 83 |   CHECK(y.getGrad() == -3);
 84 | 
 85 |   res.zeroGrad();
 86 |   res = (x +y*y)/x; // 1 + (y*y)*x^-1
 87 |   CHECK(res.getVal() == doctest::Approx(16.0/12.0));;
 88 |   res.backward();
 89 |   // -x^-2 * (y*y) = y*y/(12*12) 
 90 |   CHECK(x.getGrad() == doctest::Approx(-4.0/(12*12)));
 91 |   // 2*y/x
 92 |   CHECK(y.getGrad() == doctest::Approx(2*2.0/12.0));
 93 | }
 94 | 
 95 | 
 96 | TEST_CASE("reuse test") {
 97 |   TrackedFloat x(-1.0);
 98 |   TrackedFloat y(2.0);
 99 |   TrackedFloat res = x+y;
100 |   CHECK(res.getVal() == 1.0);
101 |   y=3.0;
102 |   res.zeroGrad(); // to redo the calculation
103 |   CHECK(res.getVal() == 2.0);
104 | }
105 | 
106 | TEST_CASE("temporaries") {
107 |   TrackedFloat x(3), y(1);
108 |   TrackedFloat res = makeFunc((x - y), SquareFunc());
109 |   CHECK(res.getVal() == 4);
110 |   res = makeFunc((x + y), SquareFunc());
111 |   CHECK(res.getVal() == 16);
112 | }
113 | 
114 | TEST_CASE("self") {
115 |   TrackedFloat x(3), y(2);
116 |   x = x/y;
117 |   CHECK(x.getVal() == doctest::Approx(3.0/2.0));
118 |   x.backward();
119 | }
120 | 
121 | TEST_CASE("reuse test grad") {
122 |   TrackedFloat x(-1.0);
123 |   TrackedFloat y(2.0);
124 |   TrackedFloat res = x+y*y;
125 |   CHECK(res.getVal() == 3.0);
126 |   res.backward();
127 |   CHECK(x.getGrad() == 1.0);
128 |   CHECK(y.getGrad() == 4.0);
129 |   res.zeroGrad();
130 |   CHECK(res.getVal() == 3.0);
131 |   res.backward();
132 |   CHECK(x.getGrad() == 1.0);
133 |   CHECK(y.getGrad() == 4.0);
134 | 
135 |   res.zeroGrad();
136 |   y=3.0;
137 | 
138 |   CHECK(res.getVal() == 8.0);
139 |   res.backward();
140 |   CHECK(x.getGrad() == 1.0);
141 |   CHECK(y.getGrad() == 6.0);
142 | 
143 | }
144 | 
145 | 
146 | TEST_CASE("relu test") {
147 |   TrackedFloat x(-1.0);
148 |   TrackedFloat y(2.0);
149 |   TrackedFloat res = makeFunc(x+y, ReluFunc());
150 |   
151 |   CHECK(res.getVal() == 1.0);
152 |   res.backward();
153 |   CHECK(x.getGrad() == 1.0);
154 |   CHECK(y.getGrad() == 1.0);
155 |   res.zeroGrad();
156 | 
157 |   y=0.0;
158 |   CHECK(res.getVal() == 0.0);
159 |   res.backward();
160 |   CHECK(x.getGrad() == 0.0);
161 |   CHECK(y.getGrad() == 0.0);
162 | }
163 | 
164 | TEST_CASE("sigmoid test") {
165 |   TrackedFloat x(-1.0);
166 |   TrackedFloat y(2.0);
167 |   TrackedFloat res = makeFunc(x+y, SigmoidFunc());
168 |   
169 |   CHECK(res.getVal() ==   MakeSigmoidFunc().func(1.0));
170 |   res.backward();
171 |   CHECK(x.getGrad() == MakeSigmoidFunc().deriv(1.0));
172 |   CHECK(y.getGrad() == MakeSigmoidFunc().deriv(1.0));
173 | 
174 |   res.zeroGrad();
175 |   x=-20;
176 |   CHECK(res.getVal() ==   MakeSigmoidFunc().func(-18.0));
177 |   res.backward();
178 |   CHECK(x.getGrad() == MakeSigmoidFunc().deriv(-18.0));
179 |   CHECK(y.getGrad() == MakeSigmoidFunc().deriv(-18.0));
180 |   
181 | }
182 | 
183 | 
184 | TEST_CASE("sigmoid advanced test") {
185 |   TrackedFloat x(-1.0);
186 |   TrackedFloat y(2.0);
187 |   TrackedFloat res = makeFunc(x*x+y*y*y, SigmoidFunc());
188 |   
189 |   CHECK(res.getVal() ==   MakeSigmoidFunc().func(9.0));
190 |   res.backward();
191 |   CHECK(x.getGrad() == -2*MakeSigmoidFunc().deriv(9.0));
192 |   CHECK(y.getGrad() == 3*4*MakeSigmoidFunc().deriv(9.0));
193 | }
194 | 
195 | TEST_CASE("sigmoid negative test") {
196 |   TrackedFloat x(-1.0);
197 |   TrackedFloat y(2.0);
198 |   TrackedFloat one(1.0);
199 |   TrackedFloat res = one-makeFunc(x*x+y*y*y, SigmoidFunc());
200 |   
201 |   CHECK(res.getVal() ==   1-MakeSigmoidFunc().func(9.0));
202 |   res.backward();
203 |   CHECK(x.getGrad() == 2*MakeSigmoidFunc().deriv(9.0));
204 |   CHECK(y.getGrad() == -3*4*MakeSigmoidFunc().deriv(9.0));
205 | }
206 | 
207 | TEST_CASE("tanh test") {
208 |   TrackedFloat x(1.0);
209 |   TrackedFloat y(1.0);
210 |   TrackedFloat res = makeFunc(x-y, TanhFunc());
211 |   
212 |   CHECK(res.getVal() == doctest::Approx(0.0));
213 |   res.backward();
214 |   CHECK(x.getGrad() == doctest::Approx(1.0));
215 |   CHECK(y.getGrad() == doctest::Approx(-1.0));
216 | 
217 |   res.zeroGrad();
218 |   x = 2;
219 |   CHECK(res.getVal() ==   doctest::Approx(tanhf(1.0)));
220 |   res.backward();
221 |   CHECK(x.getGrad() == doctest::Approx(1 - tanhf(1.0)*tanhf(1.0)));
222 |   CHECK(y.getGrad() == -x.getGrad());
223 |   
224 | }
225 | 
226 | 
227 | 
228 | TEST_CASE("vector test") {
229 |   vector<TrackedFloat> vec;
230 |   for(int n=0; n < 100; ++n)
231 |     vec.emplace_back(n);
232 | 
233 |   TrackedFloat res(0.0);
234 |   for(const auto& v : vec)
235 |     res = res + v;
236 |   
237 |   CHECK(res.getVal() == 4950);
238 |   res.backward();
239 |   CHECK(vec[0].getGrad() == 1);
240 |   CHECK(vec[10].getGrad() == 1);
241 |   CHECK(vec[20].getGrad() == 1);
242 | 
243 |   res.zeroGrad();
244 |   vec[0] = 1.0;
245 |   
246 |   CHECK(res.getVal() == 4951);  
247 | }
248 | 
249 | 
250 | TEST_CASE("typical loss func test") {
251 |   TrackedFloat result(0.75);
252 |   TrackedFloat expected(1.0);
253 |   TrackedFloat loss = (expected - result) * (expected - result);
254 | 
255 |   // expected^2 -2*expected*result +result^2
256 |   // deriv result:  -2*expected + 2 * result -> -0.5
257 |   
258 |   CHECK(loss.getVal() == (1-0.75)*(1-0.75));
259 |   loss.backward();
260 |   double grad = result.getGrad();
261 |   //  cout<<"grad: "<< grad <<endl;
262 |   loss.zeroGrad();
263 | 
264 |   result = result.getVal() - 0.1*grad;
265 |   CHECK(loss.getVal() == doctest::Approx((1.0-0.80)*(1.0-0.80)));
266 | }
267 | 
268 | 
269 | TEST_CASE("max test") {
270 | 
271 |   TrackedFloat x(2), y(3);
272 |   auto res = makeMax(x,y);
273 |   CHECK(res.getVal() == 3);
274 |   res.backward();
275 |   CHECK(x.getGrad()==0);
276 |   CHECK(y.getGrad()==1);
277 |   res.zeroGrad();
278 |   auto res2 = TrackedFloat(4)*res;
279 |   CHECK(res2.getVal() == 12);
280 |   res2.backward();
281 |   CHECK(x.getGrad()==0);
282 |   CHECK(y.getGrad()==4);
283 |   
284 | }
285 | 


--------------------------------------------------------------------------------
/hello-dl.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <random>
  3 | #include <Eigen/Dense>
  4 | #include "mnistreader.hh"
  5 | #include <map>
  6 | #include <fstream>
  7 | #include <thread>
  8 | #include "misc.hh"
  9 | 
 10 | using namespace std;
 11 | using namespace Eigen;
 12 | 
 13 | static float sigmoid(const float z)
 14 | {
 15 |   return 1.0 / (1.0 + exp(-z));
 16 | } 
 17 | 
 18 | static float gelu(const float x)
 19 | {
 20 |   constexpr float invsqrt2 = .70710678118654752440; // 1/sqrt(2)
 21 |   return 0.5*x*(1+erff(x*invsqrt2));  
 22 | }
 23 | 
 24 | template<int INPUTS, int OUTPUTS>
 25 | struct TLayer
 26 | {
 27 |   Matrix<float, OUTPUTS, INPUTS> weights, weightsGrad;
 28 |   Matrix<float, 1, OUTPUTS>  bias, biasGrad;
 29 |   std::function<float(float)> action{[](float x){return x;}};
 30 | 
 31 |   void randomize()
 32 |   {
 33 |     std::random_device rd{};
 34 |     std::mt19937 gen{rd()};
 35 |  
 36 |     std::normal_distribution<> d{0, 1};
 37 | 
 38 |     auto nd=[&gen, &d](int) {
 39 |       return (float)d(gen);
 40 |     };
 41 |     weights = Matrix<float, OUTPUTS,INPUTS>::Zero().unaryExpr(nd);
 42 |     bias = Matrix<float, 1, OUTPUTS>::Zero().unaryExpr(nd);
 43 |   }
 44 |   
 45 |   Matrix<float, 1, OUTPUTS> process(const Matrix<float, INPUTS, 1>& input) const
 46 |   {
 47 |     Matrix<float, 1, OUTPUTS> ret = weights * input;
 48 |     ret += bias;
 49 |     ret = ret.unaryExpr(action);
 50 |     return ret;
 51 |   }
 52 | 
 53 |   void doGrad(std::function<float()> lossfunc)
 54 |   {
 55 |     double h=0.001;
 56 |     double origloss = lossfunc();
 57 |     for(int r=0; r < weights.rows(); ++r) {
 58 |       for(int c=0; c < weights.cols(); ++c) {
 59 |         float oldval = weights(r,c);
 60 |         weights(r,c) += h; // tweak a bit
 61 |         double newloss = lossfunc();
 62 |         weights(r,c) = oldval;
 63 |         weightsGrad(r,c) = (newloss-origloss)/h;
 64 |       }
 65 |     }
 66 |     biasGrad.setZero();
 67 |     for(int r=0; r < bias.rows(); ++r) {
 68 |       for(int c=0; c < bias.cols(); ++c) {
 69 |         float oldval = bias(r,c);
 70 |         bias(r,c) += h; // tweak a bit
 71 |         double newloss = lossfunc();
 72 |         bias(r,c) = oldval;
 73 |         biasGrad(r,c) = (newloss-origloss)/h;
 74 |       }
 75 |     }
 76 |   }
 77 | 
 78 |   void applyGrad(float lr)
 79 |   {
 80 |     weights -= lr * weightsGrad;
 81 |     bias -= lr * biasGrad;
 82 |   }
 83 | };
 84 | 
 85 | 
 86 | typedef Matrix<float, 28*28,1> img_t;
 87 | 
 88 | void printImg(const img_t& img)
 89 | {
 90 |   cout<<"Image: "<<img.mean()<<"\n";
 91 |   for(int y=0; y < 28; ++y) {
 92 |     for(int x=0; x < 28; ++x) {
 93 |       float val = img(x*28+y);
 94 |       if(val > 0.5)
 95 |         cout<<'X';
 96 |       else if(val > 0.25)
 97 |         cout<<'*';
 98 |       else if(val > 0.125)
 99 |         cout<<'.';
100 |       else
101 |         cout<<' ';
102 |     }
103 |     cout<<'\n';
104 |   }
105 |   cout << img(28*28-1);
106 |   cout<<"\n";
107 | }
108 | 
109 | template<typename T>
110 | void scoreModel(const T& l, const MNISTReader& mntest)
111 | {
112 |   unsigned int corrects=0, wrongs=0;
113 |   int threes=0, sevens=0, threepreds=0, sevenpreds=0;
114 |   for(unsigned int i = 0 ; i < mntest.num() - 1; ++i){
115 |     int label = mntest.getLabel(i);
116 |     if(label==3)
117 |       threes++;
118 |     else if(label == 7)
119 |       sevens++;
120 |     else continue;
121 | 
122 |     img_t pic = mntest.getImageEigen(i);
123 |     MatrixXf result = l.process(pic); 
124 | 
125 |     int verdict = result(0) < 0.5 ? 3 : 7;
126 |     //    cout<<"label "<<(int)label<<" result(0) "<<result(0)<<" verdict " <<verdict<<" pic "<<pic.mean()<<endl;
127 |     //cout<<"l1bias "<<l.l1.bias<<" l2bias "<<l.l2.bias<<endl;
128 |     //    cout<<l.l1.weights.cwiseAbs().mean()<<endl;
129 |     if(verdict == label) {
130 |       corrects++;
131 |     }
132 |     else {
133 |       wrongs++;
134 |     }
135 | 
136 |     if(verdict==3)
137 |       threepreds++;
138 |     else if(verdict == 7)
139 |       sevenpreds++;
140 |   }
141 |   double perc = corrects*100.0/(corrects+wrongs);
142 |   cout<<perc<<"% correct, threes "<<threes<<" sevens "<<sevens<<" threepreds "<<threepreds<<" sevenpreds "<<sevenpreds<<"\n";
143 |   static ofstream ofs("./results.csv");
144 |   static int lcount;
145 |   if(!lcount) {
146 |     ofs<<"count,perc,threes,sevens"<<endl;
147 |   }
148 |   ofs<<lcount<<","<< perc <<","<<threepreds<<","<<sevenpreds<<endl;
149 |   lcount++;
150 |   static int perclim;
151 |   if(perc > perclim) {
152 |     ofstream ofs2("model-"+to_string(perclim)+".ppm");
153 |     ofs2<<"P6\n# example from the man page\n280 280\n255\n";
154 |     float maxval = l.weights.maxCoeff(), minval=l.weights.minCoeff();
155 |     string line;
156 |     for(int n=0; n < l.weights.cols(); ++n) {
157 |       if(!(n%28)) {
158 |         for(int rep = 0; rep < 10; ++rep)
159 |           ofs2<<line;
160 |         line.clear();
161 |       }
162 |       float val = l.weights(n);
163 |       unsigned char r{0},g{0},b{0};
164 |       if(val > 0)
165 |         r = 255 * log(1+val)/log(maxval);
166 |       else if(val < 0)
167 |         b = 255 * log(-val +1)/log(-minval);
168 |       for(int rep = 0; rep < 10; ++rep) {
169 |         line.append(1, (char)r);
170 |         line.append(1, (char)g);
171 |         line.append(1, (char)b);
172 |       }
173 | 
174 |     }
175 |     for(int rep = 0; rep < 10; ++rep)
176 |       ofs2<<line;
177 | 
178 |     perclim = (int)perc+1;
179 |   }
180 | }
181 | 
182 | int main()
183 | {
184 |   MNISTReader mn("gzip/emnist-digits-train-images-idx3-ubyte.gz", "gzip/emnist-digits-train-labels-idx1-ubyte.gz");
185 |   cout<<"Have "<<mn.num()<<" images"<<endl;
186 | 
187 |   MNISTReader mntest("gzip/emnist-digits-test-images-idx3-ubyte.gz", "gzip/emnist-digits-test-labels-idx1-ubyte.gz");
188 | 
189 |   cout<<"Start training"<<endl;
190 |   
191 |   srandom(time(0));
192 | 
193 |   TLayer<28*28, 1> l1;
194 |   l1.randomize();
195 |   l1.action = sigmoid;
196 | 
197 |   struct TwoLayers
198 |   {
199 |     TLayer<28*28, 30> l1;
200 |     TLayer<30, 1> l2;
201 |     Matrix<float, 1, 1> process(const img_t& in) const
202 |     {
203 |       //return l1.process(in);
204 |       return l2.process(l1.process(in));
205 |     }
206 |     TwoLayers()
207 |     {
208 |       l1.randomize();
209 |       l1.action = gelu;
210 |       l2.randomize();
211 |       l2.action = sigmoid;
212 |     }
213 |     
214 |     void doGrad(std::function<double()> lossfunc) 
215 |     {
216 |       l1.doGrad(lossfunc);
217 |       l2.doGrad(lossfunc);
218 |     }
219 |     
220 |     void applyGrad(float lr)
221 |     {
222 |       l1.applyGrad(lr);
223 |       l2.applyGrad(lr);
224 |     }
225 |   };
226 |   TwoLayers tl;
227 |   
228 |   vector<int> threeseven;
229 |   for(unsigned int i = 0 ; i < mn.num(); ++i) {
230 |     int label = mn.getLabel(i);
231 |     if(label==3 || label == 7)
232 |       threeseven.push_back(i);
233 |   }
234 |   cout<<"Have "<<threeseven.size()<<" threes and sevens"<<endl;
235 | 
236 |   auto& themodel = l1; // or tl
237 | 
238 |   for(;;) {
239 |     cout<<"Starting with a freshly shuffled batcher"<<endl;
240 |     Batcher batcher(threeseven);
241 |     
242 |     for(;;) {
243 |       scoreModel(themodel, mntest);
244 |       auto batch = batcher.getBatch(64);
245 |       if(batch.empty())
246 |         break;
247 | 
248 |       auto lossfunc = [&batch, &mn, &themodel]() {
249 |         double totloss = 0;
250 |         for(auto& idx : batch) {
251 |           int label = mn.getLabel(idx);
252 |           
253 |           const img_t& leImage = mn.getImageEigen(idx);
254 |           
255 |           Matrix<float, 1, 1> result = themodel.process(leImage);
256 |           
257 |           MatrixXf expected(1,1);
258 |           expected << (label == 3 ? 0.0 : 1.0);
259 |           double thisloss = abs((result-expected).mean());
260 |           totloss += thisloss;
261 |         }
262 | 
263 |         totloss /= batch.size();
264 |         return totloss;
265 |       };
266 |       
267 |       themodel.doGrad(lossfunc);
268 |       constexpr float lr = 1; // was 0.04
269 |       themodel.applyGrad(lr);
270 |     }
271 |   }
272 | }
273 | 
274 |   
275 | 


--------------------------------------------------------------------------------
/alternate/worker-convo-avx-threaded.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <vector>
  3 | #include <fstream>
  4 | #include <memory>
  5 | #include <optional>
  6 | #include "array.hh"
  7 | #include "tracked.hh"
  8 | #include "mnistreader.hh"
  9 | #include "misc.hh"
 10 | #include <string.h>
 11 | #include <mutex>
 12 | #include <fenv.h>
 13 | #include <atomic>
 14 | #include <thread>
 15 | #include "cnn-alphabet.hh"
 16 | #include "fvector.hh"
 17 | 
 18 | #include <stdio.h>
 19 | double g_lastBatchTook=0.0;
 20 | std::atomic<unsigned int> g_trained=0;
 21 | float g_lr=0.005;
 22 | std::vector<float> g_losses;
 23 | constexpr int batchsize=64;
 24 | 
 25 | using TheModel = CNNAlphabetModel<float>;
 26 | 
 27 | using namespace std;
 28 | 
 29 | ofstream g_tree; //("tree.part");
 30 | 
 31 | template<typename M, typename S>
 32 | void scoreModel(S& s, const MNISTReader& mntest, int batchno)
 33 | {
 34 |   unsigned int corrects=0, wrongs=0;
 35 |   static ofstream vcsv("validation2.csv");
 36 |   static bool notfirst;
 37 |   if(!notfirst) {
 38 |     vcsv<<"batchno,corperc,avgloss\n";
 39 |     notfirst=true;
 40 |   }
 41 |   
 42 |   M mod;
 43 |   mod.init(s);
 44 | 
 45 |   auto topo = mod.loss.getTopo();
 46 |   Batcher batcher(mntest.num());
 47 |   auto batch = batcher.getBatch(80);
 48 |   
 49 |   fvector<4> totLoss=0.0; 
 50 |   
 51 |   while(!batch.empty()) {
 52 |     mod.expected.zero();
 53 |     for(int n = 0; n < 8 ; ++n) {
 54 |       auto idx = batch.front();
 55 |       batch.pop_front();
 56 |       
 57 |       mntest.pushImage(idx, mod.img, n);
 58 |       mod.label = mntest.getLabel(idx) - 1;  // a == 1..
 59 |       
 60 |       mod.expected(0, mod.label).impl->d_val = 1; // "one hot vector"
 61 |     }
 62 |     
 63 |     totLoss += mod.loss.getVal();
 64 |     
 65 |     for(int n = 0; n < 8; ++n) {
 66 |       int verdict = mod.scores.maxValueIndexOfColumn(0);
 67 |       if(verdict == mod.label.v[n])
 68 |         corrects++;
 69 |       else
 70 |         wrongs++;
 71 |     }
 72 |     mod.loss.zeroGrad(topo);
 73 |   }
 74 |   double avgLoss = totLoss.sum()/80.0;
 75 |   double perc = 100.0*corrects/(corrects+wrongs);
 76 |   cout<<perc<<"% correct, average loss "<<avgLoss<<"\n";
 77 |   vcsv << batchno <<"," << perc << ", " << avgLoss << endl; 
 78 | }
 79 | 
 80 | int main(int argc, char** argv)
 81 | {
 82 |   feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
 83 | 
 84 |   //  std::thread graphics(graphicsThread);
 85 |   
 86 |   string kind = "letters"; // or digits
 87 |   MNISTReader mn("gzip/emnist-"+kind+"-train-images-idx3-ubyte.gz", "gzip/emnist-"+kind+"-train-labels-idx1-ubyte.gz");
 88 |   MNISTReader mntest("gzip/emnist-"+kind+"-test-images-idx3-ubyte.gz", "gzip/emnist-"+kind+"-test-labels-idx1-ubyte.gz");
 89 | 
 90 |   cout<<"Have "<<mn.num()<<" images for training, "<<mntest.num()<<" for validation"<<endl;
 91 |   
 92 |   TheModel::State s;
 93 | 
 94 |   if(argc > 1) {
 95 |     cout<<"Loading model state from "<<argv[1]<<endl;
 96 |     loadModelState(s, argv[1]);
 97 |   }
 98 | 
 99 |   auto mod = make_unique<TheModel>();
100 |   mod->init(s);
101 | 
102 |   cout<<"Getting topo"<<endl;
103 |   auto topo = mod->loss.getTopo();
104 | 
105 |   auto expected = mod->expected;
106 |   auto scores = mod->scores;
107 |   auto img = mod->img;
108 |   expected.setVariable();
109 |   mod->scores.setVariable();
110 |   mod->img.setVariable();
111 |   cout<<"Compiling work"<<endl;
112 |   auto w = mod->loss.getWork<float>(topo);  
113 |   
114 |   //  mod->scores.zero(); 
115 |   cout<<TrackedNumberImp<float>::getCount()<<" instances before clean"<<endl;
116 | 
117 | 
118 |   auto imgproj = makeProj(img, w);
119 |   auto scoresproj = makeProj(scores, w);
120 |   auto expproj = makeProj(expected, w);
121 | 
122 |   s.makeProj(w);
123 |   
124 |   auto w8 = mod->loss.getWork<fvector<8>>(topo);  
125 |   mod.reset(); // save memory, although actual cleanup only follows later
126 |   expected.reset();
127 |     
128 |   
129 |   cout<<TrackedNumberImp<float>::getCount()<<" instances, model has "<<s.size()<<" parameters, "<<topo.size()<<" nodes to be visited, worker has "<<w.work.size()<<" entries of size "<<sizeof(w.work[0])<<", total " <<w.work.size()*sizeof(w.work[0])/1000000.0<<"MB"<< endl;
130 |   cout<<w.dyns.size()<<" variable"<<endl;
131 |   
132 |   unsigned int batchno=0;
133 |   ofstream tcsv("training3.csv");
134 |   tcsv<<"batchno,corperc,avgloss"<<endl;
135 |   
136 |   for(;;) {
137 |     Batcher batcher(mn.num()); // badgerbadger
138 | 
139 |     for(;;) {
140 |       DTime dt;
141 |       dt.start();
142 |       
143 |       float totLoss = 0.0;
144 |       std::atomic<unsigned int> corrects = 0, wrongs=0;
145 | 
146 |       s.zeroGrad(); // we accumulate grads here
147 |       s.projForward(w8);
148 | 
149 |       //      if(!((tries+0) %32))
150 |       //scoreModel<TheModel, TheModel::State>(s, mntest, batchno);
151 | 
152 |       
153 |       std::atomic<int> macrob = 0;
154 |       auto workerThread = [&macrob, &mn, &imgproj, &expproj, &scoresproj, &s, &batcher, &totLoss, &corrects, &wrongs](decltype(w8)* w8ptr) {
155 |         auto& w8 = *w8ptr;
156 |         for(int ourcount = macrob++ ; ourcount < batchsize/8 ; ourcount = macrob++) { // do 8 mini batches of 8
157 |           NNArray<fvector<8>, 28, 28> img8;
158 |           NNArray<fvector<8>, 26, 1> scores8; // this should be easier to transpose
159 |           NNArray<fvector<8>, 1, 26> expected8;
160 |           
161 |           auto minibatch = batcher.getBatchLocked(8); 
162 |           if(minibatch.size() != 8)
163 |             return;
164 |           
165 |           int n = 0;
166 |           fvector<8> label = 0;
167 |           for(const auto& idx : minibatch) {
168 |             expected8.zero();
169 |             g_trained++;
170 |             mn.pushImage(idx, img8, n); 
171 |             label.v[n] = mn.getLabel(idx) - 1;  // a == 1..
172 |             expected8(0, label.v[n]).impl->d_val.v[n] = 1; // "one hot vector"
173 |           }
174 |           projForward(imgproj, img8, w8);
175 |           projForward(expproj, expected8, w8);
176 | 
177 |           float loss = w8.getResult().sum(); // runs the whole calculation
178 |           
179 |           totLoss += loss;
180 |           
181 |           projBack(scoresproj, w8, scores8);
182 |         
183 |           for(unsigned int n=0 ; n < 8; ++n) {
184 |             int verdict = scores8.getUnparallel(n).maxValueIndexOfColumn(0);
185 |             
186 |             if(ourcount == 0 && n==0) {
187 |               cout<<"Predicted: '"<< (char)('a'+verdict)<<"', actual: '"<< (char)('a' + label.v[n]) <<"': ";
188 |               if(verdict == label.v[n])
189 |                 cout<<"We got it right!"<<endl;
190 |               else
191 |                 cout<<"More learning to do.."<<endl;
192 |               
193 |               for(unsigned int x = 0 ; x < scores8.getUnparallel(n).getRows(); ++x) {
194 |                 cout << (char)('a'+x)  <<": "<< scores8.getUnparallel(n)(x, 0).getVal()<<" ";
195 |               }
196 |               cout<<endl;
197 |               printImg(img8.getUnparallel(n));
198 |             }
199 |             if(verdict == label.v[n])
200 |               corrects++;
201 |             else
202 |               wrongs++;
203 |           }
204 |           w8.backward();
205 |           s.projBackGrad(w8); // this is a where thread issues might happen
206 |           w8.zeroGrad();
207 |         }
208 |       };
209 | 
210 |       auto w8_b = w8;
211 |       std::thread t1(workerThread, &w8);
212 |       std::thread t2(workerThread, &w8_b);
213 | 
214 |       t1.join();
215 |       t2.join();
216 |       
217 |       // done with a batch
218 |       double perc = 100.0*corrects/(corrects+wrongs);
219 |       cout<<"Percent batch correct: "<<perc<<"%, average loss: "<<totLoss/batchsize <<endl;
220 |       g_losses.reserve(1024);
221 |       g_losses.push_back(totLoss/batchsize);
222 |       
223 |       tcsv << batchno <<"," << perc << ", " << totLoss / batchsize << endl; 
224 |       
225 |       double lr=g_lr/batchsize; // mnist.cpp has 0.01, plus "momentum" of 0.5, which we don't have
226 |       s.learn(lr);
227 |       s.zeroGrad();
228 |       batchno++;
229 |       g_lastBatchTook = dt.lapUsec()/1000000.0;
230 |       cout<<g_lastBatchTook<<" seconds/batch of " << batchsize << endl;;
231 |       saveModelState(s, "worker-convo-avx-threaded.state");
232 |     }
233 |   }
234 | }
235 | 


--------------------------------------------------------------------------------
/alternate/worker-convo.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <vector>
  3 | #include <fstream>
  4 | #include <memory>
  5 | #include <optional>
  6 | #include "array.hh"
  7 | #include "tracked.hh"
  8 | #include "mnistreader.hh"
  9 | #include "misc.hh"
 10 | #include <string.h>
 11 | #include <mutex>
 12 | #include <fenv.h>
 13 | #include <atomic>
 14 | #include <thread>
 15 | #include "cnn-alphabet.hh"
 16 | #include "fvector.hh"
 17 | 
 18 | 
 19 | double g_lastBatchTook=0.0;
 20 | std::atomic<unsigned int> g_trained=0;
 21 | float g_lr=0.005;
 22 | std::vector<float> g_losses;
 23 | constexpr int batchsize=128;
 24 | 
 25 | 
 26 | using TheModel = CNNAlphabetModel<float>;
 27 | 
 28 | using namespace std;
 29 | 
 30 | ofstream g_tree; //("tree.part");
 31 | 
 32 | template<typename M, typename S>
 33 | void scoreModel(S& s, const MNISTReader& mntest, int batchno)
 34 | {
 35 |   unsigned int corrects=0, wrongs=0;
 36 |   static ofstream vcsv("validation2.csv");
 37 |   static bool notfirst;
 38 |   if(!notfirst) {
 39 |     vcsv<<"batchno,corperc,avgloss\n";
 40 |     notfirst=true;
 41 |   }
 42 |   
 43 |   M mod;
 44 |   mod.init(s);
 45 | 
 46 |   auto topo = mod.loss.getTopo();
 47 |   Batcher batcher(mntest.num());
 48 |   auto batch = batcher.getBatch(80);
 49 |   
 50 |   fvector<4> totLoss=0.0; 
 51 |   
 52 |   while(!batch.empty()) {
 53 |     mod.expected.zero();
 54 |     for(int n = 0; n < 8 ; ++n) {
 55 |       auto idx = batch.front();
 56 |       batch.pop_front();
 57 |       
 58 |       mntest.pushImage(idx, mod.img, n);
 59 |       mod.label = mntest.getLabel(idx) - 1;  // a == 1..
 60 |       
 61 |       mod.expected(0, mod.label).impl->d_val = 1; // "one hot vector"
 62 |     }
 63 |     
 64 |     totLoss += mod.loss.getVal();
 65 |     
 66 |     for(int n = 0; n < 8; ++n) {
 67 |       int verdict = mod.scores.maxValueIndexOfColumn(0);
 68 |       if(verdict == mod.label.v[n])
 69 |         corrects++;
 70 |       else
 71 |         wrongs++;
 72 |     }
 73 |     mod.loss.zeroGrad(topo);
 74 |   }
 75 |   double avgLoss = totLoss.sum()/80.0;
 76 |   double perc = 100.0*corrects/(corrects+wrongs);
 77 |   cout<<perc<<"% correct, average loss "<<avgLoss<<"\n";
 78 |   vcsv << batchno <<"," << perc << ", " << avgLoss << endl; 
 79 | }
 80 | 
 81 | int main(int argc, char** argv)
 82 | {
 83 |   feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
 84 | 
 85 |   //  std::thread graphics(graphicsThread);
 86 |   
 87 |   string kind = "letters"; // or digits
 88 |   MNISTReader mn("gzip/emnist-"+kind+"-train-images-idx3-ubyte.gz", "gzip/emnist-"+kind+"-train-labels-idx1-ubyte.gz");
 89 | 
 90 |   MNISTReader mntest("gzip/emnist-"+kind+"-test-images-idx3-ubyte.gz", "gzip/emnist-"+kind+"-test-labels-idx1-ubyte.gz");
 91 | 
 92 |   cout<<"Have "<<mn.num()<<" images for training, "<<mntest.num()<<" for validation"<<endl;
 93 |   
 94 |   TheModel::State s;
 95 | 
 96 |   if(argc > 1) {
 97 |     cout<<"Loading model state from "<<argv[1]<<endl;
 98 |     loadModelState(s, argv[1]);
 99 |   }
100 | 
101 |   auto mod = make_unique<TheModel>();
102 |   mod->init(s);
103 | 
104 |   cout<<"Getting topo"<<endl;
105 |   auto topo = mod->loss.getTopo();
106 | 
107 |   auto expected = mod->expected;
108 |   auto scores = mod->scores;
109 |   auto img = mod->img;
110 |   expected.setVariable();
111 |   mod->scores.setVariable();
112 |   mod->img.setVariable();
113 |   cout<<"scores variable, needsgrad: "<<scores(0,0).impl->d_variable<<" " <<scores(0,0).impl->d_needsgrad << " mode "<< (unsigned int)scores(0,0).impl->d_mode<<endl;
114 |   cout<<"Compiling work"<<endl;
115 |   auto w = mod->loss.getWork<float>(topo);  
116 |   
117 |   //  mod->scores.zero(); 
118 |   cout<<TrackedNumberImp<float>::getCount()<<" instances before clean"<<endl;
119 |   mod.reset(); // save memory, although actual cleanup only follows later
120 |   
121 |   cout<<TrackedNumberImp<float>::getCount()<<" instances, model has "<<s.size()<<" parameters, "<<topo.size()<<" nodes to be visited, worker has "<<w.work.size()<<" entries of size "<<sizeof(w.work[0])<<", total " <<w.work.size()*sizeof(w.work[0])/1000000.0<<"MB"<< endl;
122 |   cout<<w.dyns.size()<<" variable"<<endl;
123 |   
124 |   unsigned int batchno=0;
125 |   ofstream tcsv("training2.csv");
126 |   tcsv<<"batchno,corperc,avgloss"<<endl;
127 | 
128 |   vector<decltype(w)> works(3);
129 |   for(auto& nw : works) {
130 |     cout<<"Copying\n";
131 |     nw = w;
132 |   }
133 |   
134 |   for(;;) {
135 |     Batcher batcher(mn.num()); // badgerbadger
136 | 
137 |     for(int tries=0;;++tries) {
138 |       //      if(!((tries+0) %32))
139 |       //scoreModel<TheModel, TheModel::State>(s, mntest, batchno);
140 |       DTime dt;
141 |       dt.start();
142 |       auto batch = batcher.getBatch(batchsize);
143 |       if(batch.size() != batchsize)
144 |         break;
145 | 
146 |       float totLoss = 0.0;
147 |       std::atomic<unsigned int> corrects = 0, wrongs=0;
148 | 
149 |       s.zeroGrad(); // we accumulate grads here
150 |       std::mutex lock;
151 | 
152 |       auto threadWorker = [&mn, &corrects, &wrongs, &batch, &lock, &totLoss, &img, &expected, &scores](decltype(w)* wptr) {
153 |         auto& w = *wptr;
154 |         for(;;) {
155 |           unsigned int idx;
156 |           int label;
157 |           {
158 |             DTime dt;
159 |             dt.start();
160 |             std::lock_guard<std::mutex> m(lock);
161 |             //            cout<<"Spent "<<dt.lapUsec()<<" waiting for lock\n";
162 |             dt.start();
163 |             if(batch.empty())
164 |               break;
165 |             g_trained++;
166 |             idx = batch.front();
167 |             batch.pop_front();
168 |             mn.pushImage(idx, img); // XX MUTEX
169 |             label = mn.getLabel(idx) - 1;  // a == 1..
170 |             expected.zero(); // XXX MUTEX
171 |             expected(0, label).impl->d_val = 1; // "one hot vector"
172 |             w.syncVariable(); // XXX MUTEX
173 |             //            cout<<"Spent "<<dt.lapUsec()<<" usec locked"<<endl;
174 |           }
175 |         
176 |           float loss = w.getResult(); // runs the whole calculation
177 |           
178 |           w.backward();
179 |           
180 |           decltype(scores.flatViewCol().getS()) ourscores;
181 |           int verdict;
182 |           {
183 |             std::lock_guard<std::mutex> m(lock);
184 |             totLoss += loss;
185 |             w.syncBack(); // so the scores match again
186 |             verdict = scores.maxValueIndexOfColumn(0);
187 |             ourscores = scores.flatViewCol().getS();
188 |           }
189 |           
190 |           if(corrects + wrongs < 1) {
191 |             cout<<"Predicted: '"<< (char)('a'+verdict)<<"', actual: '"<< (char)('a' + label) <<"': ";
192 |             if(verdict == label)
193 |               cout<<"We got it right!"<<endl;
194 |             else
195 |               cout<<"More learning to do.."<<endl;
196 |             cout<<"Loss: "<< loss <<", ";
197 |             //// cols, see 'flatviewcol.getS above ////
198 |             for(unsigned int x = 0 ; x < ourscores.getCols(); ++x) {
199 |               cout << (char)('a'+x)  <<": "<<ourscores(0, x)<<" ";
200 |             }
201 |             cout<<endl;
202 |             printImg(img);
203 |           }
204 |           if(verdict == label)
205 |             corrects++;
206 |           else
207 |             wrongs++;
208 |           
209 |           {
210 |             std::lock_guard<std::mutex> m(lock);
211 |             w.syncAddGrad();  // gather grads
212 |           }
213 |           w.zeroGrad(); // 16 milliseconds
214 |         }
215 |       };
216 |       threadWorker(&w);
217 |       //      std::thread t1(threadWorker, &w);
218 |       //      std::thread t2(threadWorker, &works[0]);
219 |       //      std::thread t3(threadWorker, &works[1]);
220 |       //std::thread t4(threadWorker, &works[2]);
221 |       //      t1.join(); t2.join(); //t3.join(); t4.join();
222 | 
223 |       double perc = 100.0*corrects/(corrects+wrongs);
224 |       cout<<"Percent batch correct: "<<perc<<"%, average loss: "<<totLoss/batchsize <<endl;
225 |       g_losses.reserve(1024);
226 |       g_losses.push_back(totLoss/batchsize);
227 | 
228 |       tcsv << batchno <<"," << perc << ", " << totLoss / batchsize << endl; 
229 |       
230 |       double lr=g_lr/batchsize; // mnist.cpp has 0.01, plus "momentum" of 0.5, which we don't have
231 |       s.learn(lr);
232 |       s.zeroGrad();
233 |       batchno++;
234 |       g_lastBatchTook = dt.lapUsec()/1000000.0;
235 |       cout<<g_lastBatchTook<<" seconds/batch of " << batchsize << endl;;
236 |       saveModelState(s, "worker-convo.state");
237 |     }
238 |   }
239 | }
240 | 


--------------------------------------------------------------------------------
/ext/sqlitewriter/sqlwriter.cc:
--------------------------------------------------------------------------------
  1 | #include "sqlwriter.hh"
  2 | #include <algorithm>
  3 | #include <unistd.h>
  4 | #include "sqlite3.h"
  5 | using namespace std;
  6 | 
  7 | MiniSQLite::MiniSQLite(std::string_view fname)
  8 | {
  9 |   if ( sqlite3_open(&fname[0], &d_sqlite)!=SQLITE_OK ) {
 10 |     throw runtime_error("Unable to open "+(string)fname+" for sqlite");
 11 |   }
 12 |   exec("PRAGMA journal_mode='wal'");
 13 |   sqlite3_busy_timeout(d_sqlite, 60000);
 14 | }
 15 | 
 16 | //! Get field names and types from a table
 17 | vector<pair<string,string> > MiniSQLite::getSchema(const std::string& table)
 18 | {
 19 |   vector<pair<string,string>> ret;
 20 |   
 21 |   auto rows = exec("SELECT cid,name,type FROM pragma_table_xinfo('"+table+"')");
 22 | 
 23 |   for(const auto& r : rows) {
 24 |     ret.push_back({r[1], r[2]});
 25 |   }
 26 |   sort(ret.begin(), ret.end(), [](const auto& a, const auto& b) {
 27 |     return a.first < b.first;
 28 |   });
 29 | 
 30 |   //  cout<<"returning "<<ret.size()<<" rows for table "<<table<<"\n";
 31 |   return ret;
 32 | }
 33 | 
 34 | int MiniSQLite::helperFunc(void* ptr, int cols, char** colvals, char** colnames [[maybe_unused]])
 35 | {
 36 |   vector<string> row;
 37 |   row.reserve(cols);
 38 |   for(int n=0; n < cols ; ++n)
 39 |     row.push_back(colvals[n]);
 40 |   ((MiniSQLite*)ptr)->d_rows.push_back(row);
 41 |   return 0;
 42 | }
 43 | 
 44 | vector<vector<string>> MiniSQLite::exec(std::string_view str)
 45 | {
 46 |   char *errmsg;
 47 |   std::string errstr;
 48 |   //  int (*callback)(void*,int,char**,char**)
 49 |   d_rows.clear();
 50 |   int rc = sqlite3_exec(d_sqlite, &str[0], helperFunc, this, &errmsg);
 51 |   if (rc != SQLITE_OK) {
 52 |     errstr = errmsg;
 53 |     sqlite3_free(errmsg);
 54 |     throw std::runtime_error("Error executing sqlite3 query '"+(string)str+"': "+errstr);
 55 |   }
 56 |   return d_rows; 
 57 | }
 58 | 
 59 | void MiniSQLite::bindPrep(const std::string& table, int idx, bool value) {   sqlite3_bind_int(d_stmts[table], idx, value ? 1 : 0);   }
 60 | void MiniSQLite::bindPrep(const std::string& table, int idx, int value) {   sqlite3_bind_int(d_stmts[table], idx, value);   }
 61 | void MiniSQLite::bindPrep(const std::string& table, int idx, uint32_t value) {   sqlite3_bind_int64(d_stmts[table], idx, value);   }
 62 | void MiniSQLite::bindPrep(const std::string& table, int idx, long value) {   sqlite3_bind_int64(d_stmts[table], idx, value);   }
 63 | void MiniSQLite::bindPrep(const std::string& table, int idx, unsigned long value) {   sqlite3_bind_int64(d_stmts[table], idx, value);   }
 64 | void MiniSQLite::bindPrep(const std::string& table, int idx, long long value) {   sqlite3_bind_int64(d_stmts[table], idx, value);   }
 65 | void MiniSQLite::bindPrep(const std::string& table, int idx, unsigned long long value) {   sqlite3_bind_int64(d_stmts[table], idx, value);   }
 66 | void MiniSQLite::bindPrep(const std::string& table, int idx, double value) {   sqlite3_bind_double(d_stmts[table], idx, value);   }
 67 | void MiniSQLite::bindPrep(const std::string& table, int idx, const std::string& value) {   sqlite3_bind_text(d_stmts[table], idx, value.c_str(), value.size(), SQLITE_TRANSIENT);   }
 68 | 
 69 | 
 70 | void MiniSQLite::prepare(const std::string& table, string_view str)
 71 | {
 72 |   if(d_stmts[table]) {
 73 |     sqlite3_finalize(d_stmts[table]);
 74 |     d_stmts[table] = 0;
 75 |   }
 76 |   const char* pTail;
 77 |   
 78 |   if (sqlite3_prepare_v2(d_sqlite, &str[0], -1, &d_stmts[table], &pTail ) != SQLITE_OK) {
 79 |     throw runtime_error("Unable to prepare query "+(string)str);
 80 |   }
 81 | }
 82 | 
 83 | void MiniSQLite::execPrep(const std::string& table)
 84 | {
 85 |   int rc;
 86 |   for(;;) {
 87 |     rc = sqlite3_step(d_stmts[table]); // XXX this needs to be an error checking loop
 88 |     if(rc == SQLITE_DONE)
 89 |       break;
 90 |     else
 91 |       throw runtime_error("Sqlite error: "+std::to_string(rc));
 92 |   }
 93 |   rc= sqlite3_reset(d_stmts[table]);
 94 |   if(rc != SQLITE_OK)
 95 |     throw runtime_error("Sqlite error: "+std::to_string(rc));
 96 |   sqlite3_clear_bindings(d_stmts[table]);
 97 | }
 98 | 
 99 | void MiniSQLite::begin()
100 | {
101 |   d_intransaction=true;
102 |   exec("begin");
103 | }
104 | void MiniSQLite::commit()
105 | {
106 |   d_intransaction=false;
107 |   exec("commit");
108 | }
109 | 
110 | void MiniSQLite::cycle()
111 | {
112 |   exec("commit;begin");
113 | }
114 | 
115 | bool MiniSQLite::haveTable(const string& table)
116 | {
117 |   return !getSchema(table).empty();
118 | }
119 | 
120 | 
121 | //! Add a column to a table with a certain type
122 | void MiniSQLite::addColumn(const string& table, string_view name, string_view type)
123 | {
124 |   // SECURITY PROBLEM - somehow we can't do prepared statements here
125 |   
126 |   if(!haveTable(table)) {
127 | #if SQLITE_VERSION_NUMBER >= 3037001
128 |     exec("create table if not exists '"+table+"' ( '"+(string)name+"' "+(string)type+" ) STRICT");
129 | #else
130 |     exec("create table if not exists '"+table+"' ( '"+(string)name+"' "+(string)type+" )");
131 | #endif
132 |   } else {
133 |     //    cout<<"Adding column "<<name<<" to table "<<table<<endl;
134 |     exec("ALTER table \""+table+"\" add column \""+string(name)+ "\" "+string(type));
135 |   }
136 | }
137 | 
138 | 
139 | 
140 | void SQLiteWriter::commitThread()
141 | {
142 |   int n=0;
143 |   while(!d_pleasequit) {
144 |     usleep(50000);
145 |     if(!(n%20)) {
146 |       std::lock_guard<std::mutex> lock(d_mutex);
147 |       d_db.cycle();
148 |     }
149 |     n++;
150 |   }
151 |   //  cerr<<"Thread exiting"<<endl;
152 | }
153 | 
154 | bool SQLiteWriter::haveColumn(const std::string& table, std::string_view name)
155 | {
156 |   if(d_columns[table].empty()) {
157 |     d_columns[table] = d_db.getSchema(table);
158 |   }
159 |   //  cout<<"Do we have column "<<name<<" in table "<<table<<endl;
160 |   // this could be more efficient somehow
161 |   pair<string, string> cmp{name, std::string()};
162 |   return binary_search(d_columns[table].begin(), d_columns[table].end(), cmp,
163 |                        [](const auto& a, const auto& b)
164 |                        {
165 |                          return a.first < b.first;
166 |                        });
167 | 
168 | }
169 | 
170 | 
171 | void SQLiteWriter::addValue(const initializer_list<std::pair<const char*, var_t>>& values, const std::string& table)
172 | {
173 |   addValueGeneric(table, values);
174 | }
175 | 
176 | void SQLiteWriter::addValue(const std::vector<std::pair<const char*, var_t>>& values, const std::string& table)
177 | {
178 |   addValueGeneric(table, values);
179 | }
180 | 
181 | 
182 | template<typename T>
183 | void SQLiteWriter::addValueGeneric(const std::string& table, const T& values)
184 | {
185 |   std::lock_guard<std::mutex> lock(d_mutex);
186 |   if(!d_db.isPrepared(table) || !equal(values.begin(), values.end(),
187 |                                        d_lastsig[table].cbegin(), d_lastsig[table].cend(),
188 |                             [](const auto& a, const auto& b)
189 |   {
190 |     return a.first == b;
191 |   })) {
192 |     //    cout<<"Starting a new prepared statement"<<endl;
193 |     string q = "insert into '"+table+"' (";
194 |     string qmarks;
195 |     bool first=true;
196 |     for(const auto& p : values) {
197 |       if(!haveColumn(table, p.first)) {
198 |         if(std::get_if<double>(&p.second)) {
199 |           d_db.addColumn(table, p.first, "REAL");
200 |           d_columns[table].push_back({p.first, "REAL"});
201 |         }
202 |         else if(std::get_if<string>(&p.second)) {
203 |           d_db.addColumn(table, p.first, "TEXT");
204 |           d_columns[table].push_back({p.first, "TEXT"});
205 |         } else  {
206 |           d_db.addColumn(table, p.first, "INT");
207 |           d_columns[table].push_back({p.first, "INT"});
208 |         }
209 | 
210 |         sort(d_columns[table].begin(), d_columns[table].end());
211 |       }
212 |       if(!first) {
213 |         q+=", ";
214 |         qmarks += ", ";
215 |       }
216 |       first=false;
217 |       q+="'"+string(p.first)+"'";
218 |       qmarks +="?";
219 |     }
220 |     q+= ") values ("+qmarks+")";
221 | 
222 |     d_db.prepare(table, q);
223 |     
224 |     d_lastsig[table].clear();
225 |     for(const auto& p : values)
226 |       d_lastsig[table].push_back(p.first);
227 |   }
228 |   
229 |   int n = 1;
230 |   for(const auto& p : values) {
231 |     std::visit([this, &n, &table](auto&& arg) {
232 |       d_db.bindPrep(table, n, arg);
233 |     }, p.second);
234 |     n++;
235 |   }
236 |   d_db.execPrep(table);
237 | }
238 | 
239 | MiniSQLite::~MiniSQLite()
240 | {
241 |   // needs to close down d_sqlite3
242 |   if(d_intransaction)
243 |     commit();
244 | 
245 |   for(auto& stmt: d_stmts)
246 |     if(stmt.second)
247 |       sqlite3_finalize(stmt.second);
248 |   
249 |   sqlite3_close(d_sqlite); // same
250 | }
251 | 


--------------------------------------------------------------------------------
/layers.hh:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include "array.hh"
  3 | #include <unistd.h>
  4 | #include <iostream>
  5 | #include <string.h>
  6 | 
  7 | struct LayerBase
  8 | {
  9 |   virtual void save(std::ostream& out) const = 0;
 10 |   virtual void load(std::istream& in)=0;
 11 |   virtual void learn(float lr) = 0;
 12 |   virtual void zeroGrad() = 0;
 13 |   virtual void reset();
 14 |   virtual unsigned int size() const = 0;
 15 | };
 16 | 
 17 | template<typename T, unsigned int IN, unsigned int OUT>
 18 | struct Linear : LayerBase
 19 | {
 20 |   NNArray<T, OUT, IN> d_weights;
 21 |   NNArray<T, OUT, 1> d_bias;
 22 | 
 23 |   std::array<unsigned int, decltype(d_weights)::SIZE> d_wproj;
 24 |   std::array<unsigned int, decltype(d_bias)::SIZE> d_bproj;
 25 |   Linear()
 26 |   {
 27 |     randomize();
 28 |   }
 29 |   void randomize() // "Xavier initialization"  http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf
 30 |   {
 31 |     d_weights.randomize(1.0/sqrt(IN));
 32 |     d_bias.randomize(1.0/sqrt(IN));
 33 |     d_weights.needsGrad();
 34 |     d_bias.needsGrad();
 35 |   }
 36 | 
 37 |   void reset() // "Xavier initialization"  http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf
 38 |   {
 39 |     d_weights.reset();
 40 |     d_bias.reset();
 41 | 
 42 |   }
 43 | 
 44 |   
 45 |   unsigned int size() const override
 46 |   {
 47 |     return d_weights.size() + d_bias.size();
 48 |   }
 49 |   
 50 |   void zeroGrad() override
 51 |   {
 52 |     d_weights.zeroGrad();
 53 |     d_bias.zeroGrad();
 54 |   }
 55 | 
 56 |   void addGrad(const Linear& rhs)
 57 |   {
 58 |     d_weights.addGrad(rhs.d_weights.getGrad());
 59 |     d_bias.addGrad(rhs.d_bias.getGrad());
 60 |   }
 61 | 
 62 |   void setGrad(const Linear& rhs, float divisor)
 63 |   {
 64 |     d_weights.setGradCons(rhs.d_weights.getGrad()/divisor);
 65 |     d_bias.setGradCons(rhs.d_bias.getGrad()/divisor);
 66 |   }
 67 | 
 68 |   void momGrad(const Linear& rhs, float momentum, float dampening = 0)
 69 |   {
 70 |     d_weights.setGrad(d_weights.getGrad()*momentum + rhs.d_weights.getGrad() * (1 - dampening));
 71 |     d_bias.setGrad( d_bias.getGrad() * momentum + rhs.d_bias.getGrad() * (1 - dampening));
 72 |   }
 73 |   
 74 |   auto forward(const NNArray<T, IN, 1>& in)
 75 |   {
 76 |     return d_weights * in + d_bias;
 77 |   }
 78 | 
 79 |   void learn(float lr) override
 80 |   {
 81 |     auto grad1 = d_weights.getGrad();
 82 |     grad1 *= lr;
 83 |     d_weights -= grad1;
 84 |     
 85 |     auto grad2 = d_bias.getGrad();
 86 |     grad2 *= lr;
 87 |     d_bias -= grad2;
 88 |   }
 89 | 
 90 |   void save(std::ostream& out) const override
 91 |   {
 92 |     d_weights.save(out);
 93 |     d_bias.save(out);
 94 |   }
 95 |   void load(std::istream& in) override
 96 |   {
 97 |     d_weights.load(in);
 98 |     d_bias.load(in);
 99 |   }
100 | 
101 |   template<typename W>
102 |   void makeProj(const W& w)
103 |   {
104 |     d_wproj = ::makeProj(d_weights, w);
105 |     d_bproj = ::makeProj(d_bias, w);
106 |   }
107 |   template<typename W>
108 |   void projForward(W& w) const
109 |   {
110 |     ::projForward(d_wproj, d_weights, w);
111 |     ::projForward(d_bproj, d_bias, w);
112 |   }
113 | 
114 |   template<typename W>
115 |   void projBackGrad(const W& w)
116 |   {
117 |     ::projBackGrad(d_wproj, w, d_weights);
118 |     ::projBackGrad(d_bproj, w, d_bias);
119 |   }
120 | 
121 |   
122 | };
123 | 
124 | 
125 | template<typename T, unsigned int ROWS, unsigned int COLS, unsigned int KERNEL,
126 |          unsigned int INLAYERS, unsigned int OUTLAYERS>
127 | struct Conv2d : LayerBase
128 | {
129 |   std::array<NNArray<T, KERNEL, KERNEL>, OUTLAYERS> d_filters;
130 |   std::array<NNArray<T, 1, 1>, OUTLAYERS> d_bias;
131 | 
132 |   std::array<std::array<unsigned int, KERNEL*KERNEL>, OUTLAYERS> d_fproj;
133 |   std::array<std::array<unsigned int, 1>, OUTLAYERS> d_bproj;
134 |   Conv2d()
135 |   {
136 |     randomize();
137 |   }
138 | 
139 |   void randomize()
140 |   {
141 |     for(auto& f : d_filters) {
142 |       f.randomize(sqrt(1.0/(INLAYERS*KERNEL*KERNEL)));
143 |       f.needsGrad();
144 |     }
145 |     for(auto& b : d_bias) {
146 |       b.randomize(sqrt(1.0/(INLAYERS*KERNEL*KERNEL)));
147 |       b.needsGrad();
148 |     }
149 |   }
150 | 
151 |   void reset()
152 |   {
153 |     for(auto& f : d_filters) {
154 |       f.reset();
155 |     }
156 |     for(auto& b : d_bias) {
157 |       b.reset();
158 |     }
159 |   }
160 | 
161 |   unsigned int size() const override
162 |   {
163 |     unsigned int ret = 0;
164 |     for(auto& f : d_filters)
165 |       ret += f.size();
166 |     for(auto& b : d_bias)
167 |       ret += b.size();
168 |     return ret;
169 |   }
170 |   
171 |   
172 |   void zeroGrad() override
173 |   {
174 |     for(auto& f : d_filters)
175 |       f.zeroGrad();
176 |     for(auto& b : d_bias)
177 |       b.zeroGrad();
178 |   }
179 | 
180 |   void addGrad(const Conv2d& rhs)
181 |   {
182 |     for(size_t i = 0 ; i < d_filters.size(); ++i)
183 |       d_filters[i].addGrad(rhs.d_filters[i].getGrad());
184 |     for(size_t i = 0 ; i < d_bias.size(); ++i)
185 |       d_bias[i].addGrad(rhs.d_bias[i].getGrad());
186 |   }
187 | 
188 |   void setGrad(const Conv2d& rhs, float divisor)
189 |   {
190 |     for(size_t i = 0 ; i < d_filters.size(); ++i)
191 |       d_filters[i].setGradCons(rhs.d_filters[i].getGrad()/divisor);
192 |     for(size_t i = 0 ; i < d_bias.size(); ++i)
193 |       d_bias[i].setGradCons(rhs.d_bias[i].getGrad()/divisor);
194 |   }
195 | 
196 |   void momGrad(const Conv2d& rhs, float momentum, float dampening = 0)
197 |   {
198 |     for(size_t i = 0 ; i < d_filters.size(); ++i)
199 |       d_filters[i].setGrad(  rhs.d_filters[i].getGrad() *(1-dampening) + d_filters[i].getGrad() * (momentum));
200 |     for(size_t i = 0 ; i < d_bias.size(); ++i)
201 |       d_bias[i].setGrad(  rhs.d_bias[i].getGrad()*(1-dampening) + d_bias[i].getGrad() * momentum);
202 |   }
203 |   
204 |   auto forward(NNArray<T, ROWS, COLS>& in)
205 |   {
206 |     std::array<NNArray<T, ROWS, COLS>, 1> a;
207 |     a[0] = in;
208 |     return forward(a);
209 |   }
210 |   
211 |   auto forward(std::array<NNArray<T, ROWS, COLS>, INLAYERS>& in)
212 |   {
213 |     std::array<NNArray<T, 1+ROWS-KERNEL, 1 + COLS - KERNEL>, OUTLAYERS> ret;
214 | 
215 |     // The output layers of the next convo2d have OUT filters
216 |     // these filters need to be applied to all IN input layers
217 |     // and the output is the addition of the outputs of those filters
218 |     
219 |     unsigned int ctr = 0;
220 |     for(auto& p : ret) {
221 |       p.zero();
222 |       for(auto& p2 : in)
223 |         p = p +  p2. template Convo2d<KERNEL>(d_filters.at(ctr), d_bias.at(ctr));
224 |       ctr++;
225 |     }
226 |     return ret;
227 |   }
228 | 
229 |   void learn(float lr) override
230 |   {
231 |     for(auto& v : d_filters) {
232 |       auto grad = v.getGrad();
233 |       grad *= lr;
234 |       v -= grad;
235 |     }
236 |     for(auto& v : d_bias) {
237 |       auto grad = v.getGrad();
238 |       grad *= lr;
239 |       v -= grad;
240 |     }
241 |   }
242 | 
243 |   void save(std::ostream& out) const override
244 |   {
245 |     for(const auto& w : d_filters)
246 |       w.save(out);
247 |     for(const auto& w : d_bias)
248 |       w.save(out);
249 |   }
250 |   void load(std::istream& in) override
251 |   {
252 |     for(auto& w : d_filters)
253 |       w.load(in);
254 |     for(auto& w : d_bias)
255 |       w.load(in);
256 |   }
257 | 
258 |   template<typename W>
259 |   void makeProj(const W& w)
260 |   {
261 |     for(size_t pos = 0 ; pos < d_filters.size(); ++pos)
262 |       d_fproj[pos] = ::makeProj(d_filters[pos], w);
263 |     for(size_t pos = 0 ; pos < d_bias.size(); ++pos)
264 |       d_bproj[pos] = ::makeProj(d_bias[pos], w);
265 |   }
266 | 
267 |   template<typename W>
268 |   void projForward(W& w) const
269 |   {
270 |     for(size_t pos = 0 ; pos < d_filters.size(); ++pos)
271 |       ::projForward(d_fproj[pos], d_filters[pos], w);
272 |     for(size_t pos = 0 ; pos < d_bias.size(); ++pos)
273 |       ::projForward(d_bproj[pos], d_bias[pos], w);
274 |   }
275 | 
276 |   template<typename W>
277 |   void projBackGrad(const W& w)
278 |   {
279 |     for(size_t pos = 0 ; pos < d_filters.size(); ++pos)
280 |       ::projBackGrad(d_fproj[pos], w, d_filters[pos]);
281 |     for(size_t pos = 0 ; pos < d_bias.size(); ++pos)
282 |       ::projBackGrad(d_bproj[pos], w, d_bias[pos]);
283 |   }
284 | 
285 | 
286 | };
287 | 
288 | template<typename T, unsigned int ROWS, unsigned int COLS, long unsigned int CHANNELS>
289 | auto flatten(const std::array<NNArray<T, ROWS, COLS>, CHANNELS>& in)
290 | {
291 |   NNArray<T, ROWS*COLS*CHANNELS, 1> ret;
292 |   
293 |   unsigned int ctr=0;
294 |   for(const auto& p : in) {
295 |     auto flatpart = p.flatViewRow();
296 |     for(unsigned int i = 0; i < p.getRows() * p.getCols(); ++i) 
297 |       ret(ctr++, 0) = flatpart(i, 0);
298 |   }
299 |   return ret;
300 | }
301 | 
302 | template<typename MS>
303 | void saveModelState(const MS& ms, const std::string& fname)
304 | {
305 |   std::ofstream ofs(fname+".tmp");
306 |   if(!ofs)
307 |     throw std::runtime_error("Can't save model to file "+fname+".tmp: "+strerror(errno));
308 |   ms.save(ofs);
309 |   ofs.flush();
310 |   ofs.close();
311 |   unlink(fname.c_str());
312 |   rename((fname+".tmp").c_str(), fname.c_str());
313 | }
314 | 
315 | template<typename MS>
316 | void loadModelState(MS& ms, const std::string& fname)
317 | {
318 |   std::ifstream ifs(fname);
319 |   if(!ifs)
320 |     throw std::runtime_error("Can't read model state from file "+fname+": "+strerror(errno));
321 |   ms.load(ifs);
322 | }
323 | 


--------------------------------------------------------------------------------
/alternate/worker-convo-avx.cc:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <vector>
  3 | #include <fstream>
  4 | #include <memory>
  5 | #include <optional>
  6 | #include "array.hh"
  7 | #include "tracked.hh"
  8 | #include "mnistreader.hh"
  9 | #include "misc.hh"
 10 | #include <string.h>
 11 | #include <mutex>
 12 | #include <fenv.h>
 13 | #include <atomic>
 14 | #include <thread>
 15 | #include "cnn-alphabet.hh"
 16 | #include "fvector.hh"
 17 | 
 18 | #include <stdio.h>
 19 | 
 20 | using TheModel = CNNAlphabetModel<float>;
 21 | 
 22 | using namespace std;
 23 | 
 24 | ofstream g_tree; //("tree.part");
 25 | 
 26 | std::shared_ptr<HyperParameters> g_hyper;
 27 | struct TrainingProgress g_progress;
 28 | 
 29 | template<typename W, typename IMGPROJ, typename SCOPROJ, typename EXPPROJ>
 30 | void scoreModel(W& w8, const IMGPROJ& imgproj, const SCOPROJ& scoresproj, const EXPPROJ& expproj, const MNISTReader& mntest, int batchno)
 31 | {
 32 |   unsigned int corrects=0, wrongs=0;
 33 |   static ofstream vcsv("validation2.csv");
 34 |   std::call_once([&vcsv]() {
 35 |     vcsv<<"batchno,corperc,avgloss\n";
 36 |   });
 37 |   Batcher batcher(mntest.num());
 38 | 
 39 |   
 40 |   unsigned int batchsize = 256;
 41 |   DTime dt;
 42 | 
 43 |   NNArray<fvector<8>, 28, 28> img8;
 44 |   NNArray<fvector<8>, 26, 1> scores8; // this should be easier to transpose
 45 |   NNArray<fvector<8>, 1, 26> expected8;
 46 |   cout<<"Start scoring against validation set\n";
 47 |   dt.start();
 48 |   float totLoss = 0.0;
 49 |   for(unsigned int macrobatch = 0 ; macrobatch < batchsize/8; ++macrobatch) {
 50 |     auto minibatch = batcher.getBatch(8);
 51 |     if(minibatch.size() != 8)
 52 |       return;
 53 | 
 54 |     fvector<8> label = 0;
 55 |     expected8.zero();
 56 |     img8.zero();
 57 |     for(int n =0 ; n < 8 ; ++n) {
 58 |       int idx = minibatch.at(n);
 59 |       g_progress.trained++;
 60 |       mntest.pushImage(idx, img8, n); 
 61 |       label.v[n] = mntest.getLabel(idx) - 1;  // a == 1..
 62 |       expected8(0, label.v[n]).impl->d_val.v[n] = 1; // "one hot vector"
 63 |     }
 64 |     projForward(imgproj, img8, w8);
 65 |     projForward(expproj, expected8, w8);
 66 |     float loss = w8.getResult().sum(); // runs the whole calculation
 67 |     totLoss += loss;
 68 |     projBack(scoresproj, w8, scores8);
 69 |         
 70 |     for(unsigned int n=0 ; n < 8; ++n) {
 71 |       int verdict = scores8.getUnparallel(n).maxValueIndexOfColumn(0);
 72 |       if(verdict == label.v[n])
 73 |         corrects++;
 74 |       else
 75 |         wrongs++;
 76 |     }
 77 |   }
 78 |   
 79 |   double avgLoss = totLoss/batchsize;
 80 |   double perc = 100.0*corrects/(corrects+wrongs);
 81 |   cout<<perc<<"% correct, average loss "<<avgLoss<<"\n";
 82 |   vcsv << batchno <<"," << perc << ", " << avgLoss << endl; 
 83 | }
 84 | 
 85 | int main(int argc, char** argv)
 86 | {
 87 |   //  feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
 88 | 
 89 |   auto hp = make_shared<HyperParameters>();
 90 |   hp->lr = 0.005;
 91 |   hp->batchMult = 8;
 92 |   hp->momentum = 0.5;
 93 |   g_hyper = hp;
 94 | 
 95 |   g_progress.losses.resize(1000);
 96 |   g_progress.corrects.resize(1000);
 97 |   for(auto& i : g_progress.losses)
 98 |     i=NAN;
 99 |   for(auto& i : g_progress.corrects)
100 |     i=NAN;
101 |   
102 |   std::thread graphThread(graphicsThread);
103 |   
104 |   string kind = "letters"; // or digits
105 |   MNISTReader mn("gzip/emnist-"+kind+"-train-images-idx3-ubyte.gz", "gzip/emnist-"+kind+"-train-labels-idx1-ubyte.gz");
106 | 
107 |   MNISTReader mntest("gzip/emnist-"+kind+"-test-images-idx3-ubyte.gz", "gzip/emnist-"+kind+"-test-labels-idx1-ubyte.gz");
108 | 
109 |   cout<<"Have "<<mn.num()<<" images for training, "<<mntest.num()<<" for validation"<<endl;
110 |   
111 |   TheModel::State s;
112 |   
113 | 
114 |   if(argc > 1) {
115 |     cout<<"Loading model state from "<<argv[1]<<endl;
116 |     loadModelState(s, argv[1]);
117 |   }
118 | 
119 |   auto mod = make_unique<TheModel>();
120 |   mod->init(s);
121 | 
122 |   cout<<"Getting topo"<<endl;
123 |   auto topo = mod->loss.getTopo();
124 | 
125 |   auto expected = mod->expected;
126 |   auto scores = mod->scores;
127 |   auto img = mod->img;
128 |   expected.setVariable();
129 |   mod->scores.setVariable();
130 |   mod->img.setVariable();
131 | 
132 |   cout<<"Compiling work"<<endl;
133 |   auto w = mod->loss.getWork<float>(topo);  
134 |   
135 |   //  mod->scores.zero(); 
136 |   cout<<TrackedNumberImp<float>::getCount()<<" instances before clean"<<endl;
137 | 
138 |   auto imgproj = makeProj(img, w);
139 |   auto scoresproj = makeProj(scores, w);
140 |   auto expproj = makeProj(expected, w);
141 | 
142 |   s.makeProj(w);
143 |   cout<<s.fc1.d_weights(0,0).getVal()<<endl;
144 |   TheModel::State gather = s;
145 |   cout<<s.fc1.d_weights(0,0).getVal()<<endl;
146 |   cout<<gather.fc1.d_weights(0,0).getVal()<<endl;
147 |   gather.reset(); // uncouple
148 |   cout<<s.fc1.d_weights(0,0).getVal()<<endl;
149 | 
150 |   
151 |   auto w8 = mod->loss.getWork<fvector<8>>(topo);  
152 |   mod.reset(); // save memory, although actual cleanup only follows later
153 |   expected.reset();
154 |   NNArray<fvector<8>, 28, 28> img8;
155 |   NNArray<fvector<8>, 26, 1> scores8; // this should be easier to transpose
156 |   NNArray<fvector<8>, 1, 26> expected8;
157 |     
158 |   
159 |   cout<<TrackedNumberImp<float>::getCount()<<" instances, model has "<<s.size()<<" parameters, "<<topo.size()<<" nodes to be visited, worker has "<<w8.work.size()<<" entries of size "<<sizeof(w8.work[0])<<", total " <<w8.work.size()*sizeof(w8.work[0])/1000000.0<<"MB"<< endl;
160 |   cout<<w8.dyns.size()<<" variable"<<endl;
161 |   
162 |   unsigned int batchno=0;
163 |   ofstream tcsv("training2.csv");
164 |   tcsv<<"batchno,corperc,avgloss,batchsize,lr,momentum"<<endl;
165 | 
166 |   s.zeroGrad(); 
167 |   
168 |   for(;;) { // start a new epoch
169 |     Batcher batcher(mn.num()); // badgerbadger
170 | 
171 |     for(;;) { // start a new batch
172 |       hp = g_hyper; // possibly updated parameters
173 |       
174 |       std::atomic<unsigned int> corrects = 0, wrongs=0;
175 | 
176 |       gather.zeroGrad(); // we accumulate grads here
177 |       s.projForward(w8);
178 | 
179 |       if(!(batchno%64))
180 |         scoreModel(w8, imgproj, scoresproj, expproj, mntest, batchno);
181 |       
182 |       DTime dt;
183 |       dt.start();
184 |       float totLoss = 0.0;
185 |       for(unsigned int macrobatch = 0 ; macrobatch < hp->getBatchSize()/8; ++macrobatch) {
186 |         DTime dt;
187 |         dt.start();
188 |         auto minibatch = batcher.getBatch(8);
189 |         if(minibatch.size() != 8)
190 |           goto batcherEmpty;
191 |         
192 |         
193 |         expected8.zero();
194 |         fvector<8> label = 0;
195 |         for(int n =0 ; n < 8 ; ++n) {
196 |           int idx = minibatch.at(n);
197 |           g_progress.trained++;
198 |           mn.pushImage(idx, img8, n); 
199 |           label.v[n] = mn.getLabel(idx) - 1;  // a == 1..
200 |           expected8(0, label.v[n]).impl->d_val.v[n] = 1; // "one hot vector"
201 |         }
202 |         projForward(imgproj, img8, w8);
203 |         projForward(expproj, expected8, w8);
204 |         float loss = w8.getResult().sum(); // runs the whole calculation
205 |         totLoss += loss;
206 |         projBack(scoresproj, w8, scores8);
207 |         
208 |         w8.backward();
209 |         
210 |         for(unsigned int n=0 ; n < 8; ++n) {
211 |           int verdict = scores8.getUnparallel(n).maxValueIndexOfColumn(0);
212 |           
213 |           if(corrects + wrongs < 1) {
214 |             cout<<"Predicted: '"<< (char)('a'+verdict)<<"', actual: '"<< (char)('a' + label.v[n]) <<"': ";
215 |             if(verdict == label.v[n])
216 |               cout<<"We got it right!"<<endl;
217 |             else
218 |               cout<<"More learning to do.."<<endl;
219 |             
220 |             for(unsigned int x = 0 ; x < scores8.getUnparallel(n).getRows(); ++x) {
221 |               cout << (char)('a'+x)  <<": "<< scores8.getUnparallel(n)(x, 0).getVal()<<" ";
222 |             }
223 |             cout<<endl;
224 |             printImg(img8.getUnparallel(n));
225 |           }
226 |           if(verdict == label.v[n])
227 |             corrects++;
228 |           else
229 |             wrongs++;
230 |         }
231 |         gather.projBackGrad(w8);
232 |         w8.zeroGrad();
233 |       }
234 |       // done with a whole batch
235 |       
236 |       double perc = 100.0*corrects/(corrects+wrongs);
237 |       cout<<"Percent batch correct: "<<perc<<"%, average loss: "<<totLoss/hp->getBatchSize() <<endl;
238 |       g_progress.losses[batchno % g_progress.losses.size()]=(totLoss/hp->getBatchSize());
239 |       g_progress.corrects[batchno % g_progress.losses.size()]= perc;
240 | 
241 |       for(int wo = 1 ; wo < 5; ++wo) {
242 |         g_progress.losses[(batchno+wo) % g_progress.losses.size()]=NAN;
243 |         g_progress.corrects[(batchno+wo) % g_progress.losses.size()]= NAN;
244 |       }  
245 |       
246 |       tcsv << batchno << "," << perc << ", " << totLoss / hp->getBatchSize() << ","<<hp->getBatchSize()<<","<<hp->lr<<","<<hp->momentum<<endl; 
247 |       
248 |       double lr=hp->lr/hp->getBatchSize(); // mnist.cpp has 0.01, plus "momentum" of 0.5, which we don't have
249 | 
250 |       s.momentum(gather, hp->momentum);
251 |       s.learn(lr);
252 |       //      s.zeroGrad();
253 |       batchno++;
254 |       g_progress.lastTook = dt.lapUsec()/1000000.0;
255 |       cout<<g_progress.lastTook<<" seconds/batch of " << hp->getBatchSize() <<", lr = "<<hp->lr<<", momentum = "<<hp->momentum<< endl;;
256 |       saveModelState(s, "worker-convo-avx.state");
257 |     }
258 |   batcherEmpty:
259 |     ;
260 |   }
261 | 
262 | }
263 | 


--------------------------------------------------------------------------------
/tensor-layers.hh:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | #include "tensor2.hh"
  3 | #include <unistd.h>
  4 | #include <fstream>
  5 | #include <vector>
  6 | #include "ext/sqlitewriter/sqlwriter.hh"
  7 | 
  8 | template<typename T>
  9 | struct TensorLayer
 10 | {
 11 |   TensorLayer() {} 
 12 |   TensorLayer(const TensorLayer& rhs) = delete;
 13 |   TensorLayer& operator=(const TensorLayer&) = delete;
 14 |   
 15 |   virtual void randomize() = 0;
 16 |   void save(std::ostream& out) const
 17 |   {
 18 |     for(const auto& p : d_params)
 19 |       p.ptr->save(out);
 20 |   }
 21 |   void load(std::istream& in) 
 22 |   {
 23 |     for(auto& p : d_params)
 24 |       p.ptr->load(in);
 25 |   }
 26 | 
 27 |   void learnAdam(float fact, unsigned int batchno, float alpha=0.001)
 28 |   {
 29 |     if(!batchno)
 30 |       batchno++;
 31 |     const float beta1=0.9, beta2=0.999, eps=1e-08; 
 32 |     for(auto& p : d_params) {
 33 |       if(!p.ptr->d_imp->d_adamval.m.rows()) {
 34 |         p.ptr->d_imp->d_adamval.m = p.ptr->d_imp->d_val;
 35 |         p.ptr->d_imp->d_adamval.m.setZero();
 36 |       }
 37 |       if(!p.ptr->d_imp->d_adamval.v.rows()) {
 38 |         p.ptr->d_imp->d_adamval.v = p.ptr->d_imp->d_val;
 39 |         p.ptr->d_imp->d_adamval.v.setZero();
 40 |       }
 41 | 
 42 |       typename Tensor<T>::EigenMatrix g = (fact * p.ptr->getAccumGrad()), g2;
 43 |       p.ptr->d_imp->d_adamval.m = beta1 * p.ptr->d_imp->d_adamval.m + (1.0 - beta1) * g;
 44 |       g2.array() = g.array().square();
 45 |       p.ptr->d_imp->d_adamval.v = beta2 * p.ptr->d_imp->d_adamval.v + (1.0 - beta2) * g2;// .unaryExpr([](const float& f) {
 46 |       //        return f*f;});
 47 |       //      p.ptr->d_imp->d_adamval.v = beta2 * p.ptr->d_imp->d_adamval.v + (1.0 - beta2) * g.square();
 48 | 
 49 |       typename Tensor<T>::EigenMatrix mhat = p.ptr->d_imp->d_adamval.m;
 50 |       mhat.array() /= (1.0 - powf(beta1, batchno));
 51 | 
 52 |       typename Tensor<T>::EigenMatrix vhat = p.ptr->d_imp->d_adamval.v;
 53 |       //      std::cout<< "before: "<<vhat.array()<< "\n";
 54 |       vhat.array() /= (1.0 - powf(beta2, batchno));
 55 |       //      p.ptr->d_imp->d_val.array() -= alpha * p.ptr->d_imp->d_adamval.m.array() / (p.ptr->d_imp->d_adamval.v.array().sqrt() + eps);
 56 |       p.ptr->d_imp->d_val.array() -= alpha * mhat.array() / (vhat.array().sqrt() + eps);
 57 |     }
 58 |   }
 59 | 
 60 |   
 61 |   void learn(float lr, float momentum) 
 62 |   {
 63 |     for(auto& p : d_params) {
 64 |       auto grad1 = (momentum * p.ptr->getAccumGrad()).eval(); // THIS IS THE MOMENTUM
 65 |       if(p.ptr->getPrevAccumGrad().rows())
 66 |         grad1 += p.ptr->getPrevAccumGrad();
 67 |       grad1 *= lr;
 68 |       *p.ptr -= grad1;
 69 |     }
 70 |   }
 71 |   // emit all parameters to SQL
 72 |   void emit(SQLiteWriter& sqw, unsigned int startID, unsigned int& pos, unsigned int batchno, unsigned int batchsize, std::string layername)
 73 |   {
 74 |     auto emit = [&sqw, &startID, &pos, &batchno, &batchsize](const auto& t, std::string_view name, std::string_view subname, unsigned int idx) {
 75 |       for(unsigned int r = 0 ; r < t.d_imp->d_val.rows(); ++r) {
 76 |         for(unsigned int c = 0 ; c < t.d_imp->d_val.cols(); ++c) {
 77 |           sqw.addValue({
 78 |               {"batchno", batchno},
 79 |               {"pos", pos},
 80 |               {"val", t.d_imp->d_val(r,c)},
 81 |               {"grad", t.d_imp->d_accumgrads(r,c)/batchsize},
 82 |               {"idx", idx},
 83 |               {"row", r},
 84 |               {"col", c},
 85 |               {"name", (std::string)name},
 86 |               {"subname", (std::string)subname},
 87 |               {"startID", startID}});
 88 |           ++pos;
 89 |         }
 90 |       }
 91 |     };
 92 |     // iterate over params
 93 |     for(const auto& p : d_params)
 94 |       emit(*p.ptr, layername, p.subname, p.idx);
 95 |   }
 96 | 
 97 |   struct Param
 98 |   {
 99 |     Tensor<T>* ptr;
100 |     std::string subname;
101 |     unsigned int idx=0;
102 |   };
103 |   std::vector<Param> d_params;
104 | };
105 | 
106 | template<typename T, unsigned int IN, unsigned int OUT>
107 | struct Linear : public TensorLayer<T>
108 | {
109 |   Tensor<T> d_weights{OUT, IN};
110 |   Tensor<T> d_bias{OUT, 1};
111 |   
112 |   Linear()
113 |   {
114 |     randomize();
115 |     this->d_params = {{&d_weights, "weights"},  {&d_bias, "bias"}};
116 |   }
117 |   void randomize() override// "Xavier initialization"  http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf
118 |   {
119 |     d_weights.randomize(1.0/sqrt(IN));
120 |     d_bias.randomize(1.0/sqrt(IN));
121 |   }
122 | 
123 |   auto forward(const Tensor<T>& in)
124 |   {
125 |     return d_weights * in + d_bias;
126 |   }
127 | 
128 |   auto SquaredWeightsSum()
129 |   {
130 |     return makeFunction<SquareFunc>(d_weights).sum();
131 |   }
132 | 
133 | };
134 | 
135 | 
136 | template<typename T, unsigned int ROWS, unsigned int COLS, unsigned int KERNEL,
137 |          unsigned int INLAYERS, unsigned int OUTLAYERS>
138 | struct Conv2d : TensorLayer<T>
139 | {
140 |   std::array<Tensor<T>, INLAYERS*OUTLAYERS> d_filters;
141 |   std::array<Tensor<T>, OUTLAYERS> d_bias;
142 | 
143 |   Conv2d()
144 |   {
145 |     unsigned int idx=0;
146 |     for(auto& f : d_filters) {
147 |       f = Tensor<T>(KERNEL, KERNEL);
148 |       this->d_params.push_back({&f, "filter", idx});
149 |       ++idx;
150 |     }
151 |     idx=0;
152 |     for(auto& b : d_bias) {
153 |       b = Tensor<T>(1,1);
154 |       this->d_params.push_back({&b, "bias", idx});
155 |       ++idx;
156 |     }
157 |         
158 |     randomize();
159 |   }
160 | 
161 |   void randomize()
162 |   {
163 |     for(auto& f : d_filters) 
164 |       f.randomize(sqrt(1.0/(INLAYERS*KERNEL*KERNEL)));
165 |     for(auto& b : d_bias) 
166 |       b.randomize(sqrt(1.0/(INLAYERS*KERNEL*KERNEL)));
167 |   }
168 |   
169 |   auto forward(Tensor<T>& in) // convenience
170 |   {
171 |     std::array<Tensor<T>, 1> a;
172 |     a[0] = in;
173 |     return forward(a);
174 |   }
175 |   
176 |   auto forward(std::array<Tensor<T>, INLAYERS>& in)
177 |   {
178 |     std::array<Tensor<T>, OUTLAYERS> ret;
179 | 
180 |     for(auto& o : ret)
181 |       o = Tensor<T>(1+ROWS-KERNEL, 1 + COLS - KERNEL);
182 |     
183 |     // The output layers of the next convo2d have OUT filters
184 |     // these filters need to be applied to all IN input layers
185 |     // and the output is the addition of the outputs of those filters
186 |     //    https://d2l.ai/chapter_convolutional-neural-networks/channels.html
187 |     unsigned int ctr = 0, bctr=0;
188 |     for(auto& p : ret) { // outlayers long
189 |       p.zero();
190 |       
191 |       for(auto& p2 : in) {
192 |         p = p +  p2.makeConvo(KERNEL, d_filters.at(ctr), d_bias.at(bctr));
193 |         ctr++;
194 |       }
195 |       bctr++;
196 |     }
197 |     return ret;
198 |   }
199 | 
200 |   auto SquaredWeightsSum()
201 |   {
202 |     Tensor<T> ret(1,1);
203 |     ret(0,0)=0;
204 |     for(auto& f : d_filters) 
205 |       ret = ret + makeFunction<SquareFunc>(f).sum();
206 |     return ret;
207 |   }
208 | };
209 |   
210 | template<typename T, std::size_t LAYERS>
211 | auto Max2dfw(std::array<Tensor<T>, LAYERS>& in, int kernel)
212 | {
213 |   std::array<Tensor<T>, LAYERS> ret;
214 | 
215 |   for(unsigned int ctr = 0 ; ctr < LAYERS; ++ctr)
216 |     ret[ctr] = in[ctr].makeMax2d(kernel);
217 |   return ret;
218 | }
219 |   
220 | template<typename T>
221 | struct ModelState
222 | {
223 |   ModelState() {}
224 |   ModelState(const ModelState&) = delete;
225 |   ModelState& operator=(const ModelState& rhs) = delete;
226 |   void randomize()
227 |   {
228 |     for(auto& m : d_members)
229 |       m.first->randomize();
230 |   }
231 |   void learn(float lr, float momentum=0)
232 |   {
233 |     for(auto& m : d_members)
234 |       m.first->learn(lr, momentum);
235 |   }
236 |   void learnAdam(float fact, unsigned int batchno, float alpha=0.001)
237 |   {
238 |     for(auto& m : d_members)
239 |       m.first->learnAdam(fact, batchno, alpha);
240 |   }
241 | 
242 |   void save(std::ostream& out) const
243 |   {
244 |     for(const auto& m : d_members)
245 |       m.first->save(out);
246 |   }
247 |   void load(std::istream& in)
248 |   {
249 |     for(auto& m : d_members)
250 |       m.first->load(in);
251 |   }
252 | 
253 |   std::vector<std::pair<TensorLayer<T>*, std::string>> d_members;
254 | 
255 |   // ask all members of a model to emit themselves for logging
256 |   void emit(SQLiteWriter& sqw, unsigned int startID, unsigned int batchno, unsigned int batchsize)
257 |   {
258 |     unsigned int pos = 0;
259 |     for(const auto& m : d_members) {
260 |       m.first->emit(sqw, startID, pos, batchno, batchsize, m.second);
261 |     }
262 |   }
263 | };
264 | 
265 | 
266 | template<typename MS>
267 | void saveModelState(const MS& ms, const std::string& fname)
268 | {
269 |   std::ofstream ofs(fname+".tmp");
270 |   if(!ofs)
271 |     throw std::runtime_error("Can't save model to file "+fname+".tmp: "+strerror(errno));
272 |   std::array<float, 2>v={12345.0, 67890.0}; // magic begin
273 |   ofs.write((const char*)&v[0], 2*sizeof(float));
274 |   ms.save(ofs);
275 |   v={9876.0,54321.0}; // magic end
276 |   ofs.write((const char*)&v[0], 2*sizeof(float));
277 |   ofs.flush();
278 |   ofs.close();
279 |   unlink(fname.c_str());
280 |   rename((fname+".tmp").c_str(), fname.c_str());
281 | }
282 | 
283 | template<typename MS>
284 | void loadModelState(MS& ms, const std::string& fname)
285 | {
286 |   std::ifstream ifs(fname);
287 |   if(!ifs)
288 |     throw std::runtime_error("Can't read model state from file "+fname+": "+strerror(errno));
289 |   float v[2]={};
290 |   ifs.read((char*)v, 2*sizeof(float));
291 |   if(v[0] != 12345 || v[1] != 67890)
292 |     throw std::runtime_error("Model state has wrong begin magic, "+std::to_string(v[0])+", "+std::to_string(v[1]));
293 |   ms.load(ifs);
294 |   ifs.read((char*)v, 2*sizeof(float));
295 |   if(v[0] != 9876 || v[1] != 54321)
296 |     throw std::runtime_error("Model state has wrong end magic, "+std::to_string(v[0])+", "+std::to_string(v[1]));
297 | 
298 | }
299 | 


--------------------------------------------------------------------------------