├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── Tinn.c ├── Tinn.h ├── img └── logo.PNG └── test.c /.gitignore: -------------------------------------------------------------------------------- 1 | *.dat* 2 | *.txt 3 | *.o 4 | *.d 5 | test 6 | *.tinn 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Gustav Louw 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BIN = test 2 | 3 | CFLAGS = -std=c99 -Wall -Wextra -pedantic -Ofast -flto -march=native 4 | 5 | LDFLAGS = -lm 6 | 7 | CC = gcc 8 | 9 | SRC = test.c Tinn.c 10 | 11 | all: 12 | $(CC) -o $(BIN) $(SRC) $(CFLAGS) $(LDFLAGS) 13 | 14 | run: 15 | ./$(BIN) 16 | 17 | clean: 18 | rm -f $(BIN) 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](img/logo.PNG) 2 | 3 | Tinn (Tiny Neural Network) is a 200 line dependency free neural network library written in C99. 4 | 5 | For a demo on how to learn hand written digits, get some training data: 6 | 7 | wget http://archive.ics.uci.edu/ml/machine-learning-databases/semeion/semeion.data 8 | 9 | make; ./test 10 | 11 | The training data consists of hand written digits written both slowly and quickly. 12 | Each line in the data set corresponds to one handwritten digit. Each digit is 16x16 pixels in size 13 | giving 256 inputs to the neural network. 14 | 15 | At the end of the line 10 digits signify the hand written digit: 16 | 17 | 0: 1 0 0 0 0 0 0 0 0 0 18 | 1: 0 1 0 0 0 0 0 0 0 0 19 | 2: 0 0 1 0 0 0 0 0 0 0 20 | 3: 0 0 0 1 0 0 0 0 0 0 21 | 4: 0 0 0 0 1 0 0 0 0 0 22 | ... 23 | 9: 0 0 0 0 0 0 0 0 0 1 24 | 25 | This gives 10 outputs to the neural network. The test program will output the 26 | accuracy for each digit. Expect above 99% accuracy for the correct digit, and 27 | less that 0.1% accuracy for the other digits. 28 | 29 | ## Features 30 | 31 | * Portable - Runs where a C99 or C++98 compiler is present. 32 | 33 | * Sigmoidal activation. 34 | 35 | * One hidden layer. 36 | 37 | ## Tips 38 | 39 | * Tinn will never use more than the C standard library. 40 | 41 | * Tinn is great for embedded systems. Train a model on your powerful desktop and load 42 | it onto a microcontroller and use the analog to digital converter to predict real time events. 43 | 44 | * The Tinn source code will always be less than 200 lines. Functions externed in the Tinn header 45 | are protected with the _xt_ namespace standing for _externed tinn_. 46 | 47 | * Tinn can easily be multi-threaded with a bit of ingenuity but the master branch will remain 48 | single threaded to aid development for embedded systems. 49 | 50 | * Tinn does not seed the random number generator. Do not forget to do so yourself. 51 | 52 | * Always shuffle your input data. Shuffle again after every training iteration. 53 | 54 | * Get greater training accuracy by annealing your learning rate. For instance, multiply 55 | your learning rate by 0.99 every training iteration. This will zero in on a good learning minima. 56 | 57 | ## Disclaimer 58 | 59 | Tinn is a practice in minimalism. 60 | 61 | Tinn is not a fully featured neural network C library like Kann, or Genann: 62 | 63 | https://github.com/attractivechaos/kann 64 | 65 | https://github.com/codeplea/genann 66 | 67 | ## Ports 68 | 69 | Rust: https://github.com/dvdplm/rustinn 70 | 71 | ## Other 72 | 73 | [A Tutorial using Tinn NN and CTypes](https://medium.com/@cknorow/creating-a-python-interface-to-a-c-library-a-tutorial-using-tinn-nn-d935707dd225) 74 | 75 | [Tiny Neural Network Library in 200 Lines of Code](https://hackaday.com/2018/04/08/tiny-neural-network-library-in-200-lines-of-code/) 76 | -------------------------------------------------------------------------------- /Tinn.c: -------------------------------------------------------------------------------- 1 | #include "Tinn.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | // Computes error. 9 | static float err(const float a, const float b) 10 | { 11 | return 0.5f * (a - b) * (a - b); 12 | } 13 | 14 | // Returns partial derivative of error function. 15 | static float pderr(const float a, const float b) 16 | { 17 | return a - b; 18 | } 19 | 20 | // Computes total error of target to output. 21 | static float toterr(const float* const tg, const float* const o, const int size) 22 | { 23 | float sum = 0.0f; 24 | for(int i = 0; i < size; i++) 25 | sum += err(tg[i], o[i]); 26 | return sum; 27 | } 28 | 29 | // Activation function. 30 | static float act(const float a) 31 | { 32 | return 1.0f / (1.0f + expf(-a)); 33 | } 34 | 35 | // Returns partial derivative of activation function. 36 | static float pdact(const float a) 37 | { 38 | return a * (1.0f - a); 39 | } 40 | 41 | // Returns floating point random from 0.0 - 1.0. 42 | static float frand() 43 | { 44 | return rand() / (float) RAND_MAX; 45 | } 46 | 47 | // Performs back propagation. 48 | static void bprop(const Tinn t, const float* const in, const float* const tg, float rate) 49 | { 50 | for(int i = 0; i < t.nhid; i++) 51 | { 52 | float sum = 0.0f; 53 | // Calculate total error change with respect to output. 54 | for(int j = 0; j < t.nops; j++) 55 | { 56 | const float a = pderr(t.o[j], tg[j]); 57 | const float b = pdact(t.o[j]); 58 | sum += a * b * t.x[j * t.nhid + i]; 59 | // Correct weights in hidden to output layer. 60 | t.x[j * t.nhid + i] -= rate * a * b * t.h[i]; 61 | } 62 | // Correct weights in input to hidden layer. 63 | for(int j = 0; j < t.nips; j++) 64 | t.w[i * t.nips + j] -= rate * sum * pdact(t.h[i]) * in[j]; 65 | } 66 | } 67 | 68 | // Performs forward propagation. 69 | static void fprop(const Tinn t, const float* const in) 70 | { 71 | // Calculate hidden layer neuron values. 72 | for(int i = 0; i < t.nhid; i++) 73 | { 74 | float sum = 0.0f; 75 | for(int j = 0; j < t.nips; j++) 76 | sum += in[j] * t.w[i * t.nips + j]; 77 | t.h[i] = act(sum + t.b[0]); 78 | } 79 | // Calculate output layer neuron values. 80 | for(int i = 0; i < t.nops; i++) 81 | { 82 | float sum = 0.0f; 83 | for(int j = 0; j < t.nhid; j++) 84 | sum += t.h[j] * t.x[i * t.nhid + j]; 85 | t.o[i] = act(sum + t.b[1]); 86 | } 87 | } 88 | 89 | // Randomizes tinn weights and biases. 90 | static void wbrand(const Tinn t) 91 | { 92 | for(int i = 0; i < t.nw; i++) t.w[i] = frand() - 0.5f; 93 | for(int i = 0; i < t.nb; i++) t.b[i] = frand() - 0.5f; 94 | } 95 | 96 | // Returns an output prediction given an input. 97 | float* xtpredict(const Tinn t, const float* const in) 98 | { 99 | fprop(t, in); 100 | return t.o; 101 | } 102 | 103 | // Trains a tinn with an input and target output with a learning rate. Returns target to output error. 104 | float xttrain(const Tinn t, const float* const in, const float* const tg, float rate) 105 | { 106 | fprop(t, in); 107 | bprop(t, in, tg, rate); 108 | return toterr(tg, t.o, t.nops); 109 | } 110 | 111 | // Constructs a tinn with number of inputs, number of hidden neurons, and number of outputs 112 | Tinn xtbuild(const int nips, const int nhid, const int nops) 113 | { 114 | Tinn t; 115 | // Tinn only supports one hidden layer so there are two biases. 116 | t.nb = 2; 117 | t.nw = nhid * (nips + nops); 118 | t.w = (float*) calloc(t.nw, sizeof(*t.w)); 119 | t.x = t.w + nhid * nips; 120 | t.b = (float*) calloc(t.nb, sizeof(*t.b)); 121 | t.h = (float*) calloc(nhid, sizeof(*t.h)); 122 | t.o = (float*) calloc(nops, sizeof(*t.o)); 123 | t.nips = nips; 124 | t.nhid = nhid; 125 | t.nops = nops; 126 | wbrand(t); 127 | return t; 128 | } 129 | 130 | // Saves a tinn to disk. 131 | void xtsave(const Tinn t, const char* const path) 132 | { 133 | FILE* const file = fopen(path, "w"); 134 | // Save header. 135 | fprintf(file, "%d %d %d\n", t.nips, t.nhid, t.nops); 136 | // Save biases and weights. 137 | for(int i = 0; i < t.nb; i++) fprintf(file, "%f\n", (double) t.b[i]); 138 | for(int i = 0; i < t.nw; i++) fprintf(file, "%f\n", (double) t.w[i]); 139 | fclose(file); 140 | } 141 | 142 | // Loads a tinn from disk. 143 | Tinn xtload(const char* const path) 144 | { 145 | FILE* const file = fopen(path, "r"); 146 | int nips = 0; 147 | int nhid = 0; 148 | int nops = 0; 149 | // Load header. 150 | fscanf(file, "%d %d %d\n", &nips, &nhid, &nops); 151 | // Build a new tinn. 152 | const Tinn t = xtbuild(nips, nhid, nops); 153 | // Load bias and weights. 154 | for(int i = 0; i < t.nb; i++) fscanf(file, "%f\n", &t.b[i]); 155 | for(int i = 0; i < t.nw; i++) fscanf(file, "%f\n", &t.w[i]); 156 | fclose(file); 157 | return t; 158 | } 159 | 160 | // Frees object from heap. 161 | void xtfree(const Tinn t) 162 | { 163 | free(t.w); 164 | free(t.b); 165 | free(t.h); 166 | free(t.o); 167 | } 168 | 169 | // Prints an array of floats. Useful for printing predictions. 170 | void xtprint(const float* arr, const int size) 171 | { 172 | for(int i = 0; i < size; i++) 173 | printf("%f ", (double) arr[i]); 174 | printf("\n"); 175 | } 176 | -------------------------------------------------------------------------------- /Tinn.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | typedef struct 4 | { 5 | // All the weights. 6 | float* w; 7 | // Hidden to output layer weights. 8 | float* x; 9 | // Biases. 10 | float* b; 11 | // Hidden layer. 12 | float* h; 13 | // Output layer. 14 | float* o; 15 | // Number of biases - always two - Tinn only supports a single hidden layer. 16 | int nb; 17 | // Number of weights. 18 | int nw; 19 | // Number of inputs. 20 | int nips; 21 | // Number of hidden neurons. 22 | int nhid; 23 | // Number of outputs. 24 | int nops; 25 | } 26 | Tinn; 27 | 28 | float* xtpredict(Tinn, const float* in); 29 | 30 | float xttrain(Tinn, const float* in, const float* tg, float rate); 31 | 32 | Tinn xtbuild(int nips, int nhid, int nops); 33 | 34 | void xtsave(Tinn, const char* path); 35 | 36 | Tinn xtload(const char* path); 37 | 38 | void xtfree(Tinn); 39 | 40 | void xtprint(const float* arr, const int size); 41 | -------------------------------------------------------------------------------- /img/logo.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/glouw/tinn/815225a8f11c7aff2f3d008cb19980f40dc60de6/img/logo.PNG -------------------------------------------------------------------------------- /test.c: -------------------------------------------------------------------------------- 1 | #include "Tinn.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | // Data object. 8 | typedef struct 9 | { 10 | // 2D floating point array of input. 11 | float** in; 12 | // 2D floating point array of target. 13 | float** tg; 14 | // Number of inputs to neural network. 15 | int nips; 16 | // Number of outputs to neural network. 17 | int nops; 18 | // Number of rows in file (number of sets for neural network). 19 | int rows; 20 | } 21 | Data; 22 | 23 | // Returns the number of lines in a file. 24 | static int lns(FILE* const file) 25 | { 26 | int ch = EOF; 27 | int lines = 0; 28 | int pc = '\n'; 29 | while((ch = getc(file)) != EOF) 30 | { 31 | if(ch == '\n') 32 | lines++; 33 | pc = ch; 34 | } 35 | if(pc != '\n') 36 | lines++; 37 | rewind(file); 38 | return lines; 39 | } 40 | 41 | // Reads a line from a file. 42 | static char* readln(FILE* const file) 43 | { 44 | int ch = EOF; 45 | int reads = 0; 46 | int size = 128; 47 | char* line = (char*) malloc((size) * sizeof(char)); 48 | while((ch = getc(file)) != '\n' && ch != EOF) 49 | { 50 | line[reads++] = ch; 51 | if(reads + 1 == size) 52 | line = (char*) realloc((line), (size *= 2) * sizeof(char)); 53 | } 54 | line[reads] = '\0'; 55 | return line; 56 | } 57 | 58 | // New 2D array of floats. 59 | static float** new2d(const int rows, const int cols) 60 | { 61 | float** row = (float**) malloc((rows) * sizeof(float*)); 62 | for(int r = 0; r < rows; r++) 63 | row[r] = (float*) malloc((cols) * sizeof(float)); 64 | return row; 65 | } 66 | 67 | // New data object. 68 | static Data ndata(const int nips, const int nops, const int rows) 69 | { 70 | const Data data = { 71 | new2d(rows, nips), new2d(rows, nops), nips, nops, rows 72 | }; 73 | return data; 74 | } 75 | 76 | // Gets one row of inputs and outputs from a string. 77 | static void parse(const Data data, char* line, const int row) 78 | { 79 | const int cols = data.nips + data.nops; 80 | for(int col = 0; col < cols; col++) 81 | { 82 | const float val = atof(strtok(col == 0 ? line : NULL, " ")); 83 | if(col < data.nips) 84 | data.in[row][col] = val; 85 | else 86 | data.tg[row][col - data.nips] = val; 87 | } 88 | } 89 | 90 | // Frees a data object from the heap. 91 | static void dfree(const Data d) 92 | { 93 | for(int row = 0; row < d.rows; row++) 94 | { 95 | free(d.in[row]); 96 | free(d.tg[row]); 97 | } 98 | free(d.in); 99 | free(d.tg); 100 | } 101 | 102 | // Randomly shuffles a data object. 103 | static void shuffle(const Data d) 104 | { 105 | for(int a = 0; a < d.rows; a++) 106 | { 107 | const int b = rand() % d.rows; 108 | float* ot = d.tg[a]; 109 | float* it = d.in[a]; 110 | // Swap output. 111 | d.tg[a] = d.tg[b]; 112 | d.tg[b] = ot; 113 | // Swap input. 114 | d.in[a] = d.in[b]; 115 | d.in[b] = it; 116 | } 117 | } 118 | 119 | // Parses file from path getting all inputs and outputs for the neural network. Returns data object. 120 | static Data build(const char* path, const int nips, const int nops) 121 | { 122 | FILE* file = fopen(path, "r"); 123 | if(file == NULL) 124 | { 125 | printf("Could not open %s\n", path); 126 | printf("Get it from the machine learning database: "); 127 | printf("wget http://archive.ics.uci.edu/ml/machine-learning-databases/semeion/semeion.data\n"); 128 | exit(1); 129 | } 130 | const int rows = lns(file); 131 | Data data = ndata(nips, nops, rows); 132 | for(int row = 0; row < rows; row++) 133 | { 134 | char* line = readln(file); 135 | parse(data, line, row); 136 | free(line); 137 | } 138 | fclose(file); 139 | return data; 140 | } 141 | 142 | // Learns and predicts hand written digits with 98% accuracy. 143 | int main() 144 | { 145 | // Tinn does not seed the random number generator. 146 | srand(time(0)); 147 | // Input and output size is harded coded here as machine learning 148 | // repositories usually don't include the input and output size in the data itself. 149 | const int nips = 256; 150 | const int nops = 10; 151 | // Hyper Parameters. 152 | // Learning rate is annealed and thus not constant. 153 | // It can be fine tuned along with the number of hidden layers. 154 | // Feel free to modify the anneal rate. 155 | // The number of iterations can be changed for stronger training. 156 | float rate = 1.0f; 157 | const int nhid = 28; 158 | const float anneal = 0.99f; 159 | const int iterations = 128; 160 | // Load the training set. 161 | const Data data = build("semeion.data", nips, nops); 162 | // Train, baby, train. 163 | const Tinn tinn = xtbuild(nips, nhid, nops); 164 | for(int i = 0; i < iterations; i++) 165 | { 166 | shuffle(data); 167 | float error = 0.0f; 168 | for(int j = 0; j < data.rows; j++) 169 | { 170 | const float* const in = data.in[j]; 171 | const float* const tg = data.tg[j]; 172 | error += xttrain(tinn, in, tg, rate); 173 | } 174 | printf("error %.12f :: learning rate %f\n", 175 | (double) error / data.rows, 176 | (double) rate); 177 | rate *= anneal; 178 | } 179 | // This is how you save the neural network to disk. 180 | xtsave(tinn, "saved.tinn"); 181 | xtfree(tinn); 182 | // This is how you load the neural network from disk. 183 | const Tinn loaded = xtload("saved.tinn"); 184 | // Now we do a prediction with the neural network we loaded from disk. 185 | // Ideally, we would also load a testing set to make the prediction with, 186 | // but for the sake of brevity here we just reuse the training set from earlier. 187 | // One data set is picked at random (zero index of input and target arrays is enough 188 | // as they were both shuffled earlier). 189 | const float* const in = data.in[0]; 190 | const float* const tg = data.tg[0]; 191 | const float* const pd = xtpredict(loaded, in); 192 | // Prints target. 193 | xtprint(tg, data.nops); 194 | // Prints prediction. 195 | xtprint(pd, data.nops); 196 | // All done. Let's clean up. 197 | xtfree(loaded); 198 | dfree(data); 199 | return 0; 200 | } 201 | --------------------------------------------------------------------------------