├── BinarizedMNISTData.zip ├── LICENSE ├── MNISTDemoBits.c ├── MultiClassTsetlinMachineBits.c ├── MultiClassTsetlinMachineBits.h ├── MultiClassTsetlinMachineConfig.h ├── README.md ├── TsetlinMachineBits.c ├── TsetlinMachineBits.h ├── TsetlinMachineBitsConfig.h ├── fast_rand.h └── makefile /BinarizedMNISTData.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cair/fast-tsetlin-machine-with-mnist-demo/6d317dddcdb610c23deb89018d570bfc1b225657/BinarizedMNISTData.zip -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Ole-Christoffer Granmo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MNISTDemoBits.c: -------------------------------------------------------------------------------- 1 | #include "MultiClassTsetlinMachineBits.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #define EPOCHS 400 9 | #define NUMBER_OF_TRAINING_EXAMPLES 60000 10 | #define NUMBER_OF_TEST_EXAMPLES 10000 11 | 12 | unsigned int X_train[NUMBER_OF_TRAINING_EXAMPLES][LA_CHUNKS]; 13 | int y_train[NUMBER_OF_TRAINING_EXAMPLES]; 14 | 15 | unsigned int X_test[NUMBER_OF_TEST_EXAMPLES][LA_CHUNKS]; 16 | int y_test[NUMBER_OF_TEST_EXAMPLES]; 17 | 18 | unsigned int X_training_2[NUMBER_OF_TEST_EXAMPLES][LA_CHUNKS]; 19 | int y_training_2[NUMBER_OF_TEST_EXAMPLES]; 20 | 21 | void read_file(void) 22 | { 23 | FILE * fp; 24 | char * line = NULL; 25 | size_t len = 0; 26 | 27 | const char *s = " "; 28 | char *token = NULL; 29 | 30 | // Training Dataset 31 | 32 | for (int i = 0; i < NUMBER_OF_TRAINING_EXAMPLES; i++) { 33 | for (int j = 0; j < LA_CHUNKS; j++) { 34 | X_train[i][j] = 0; 35 | } 36 | } 37 | 38 | fp = fopen("MNISTTraining.txt", "r"); 39 | if (fp == NULL) { 40 | printf("Error opening\n"); 41 | exit(EXIT_FAILURE); 42 | } 43 | 44 | for (int i = 0; i < NUMBER_OF_TRAINING_EXAMPLES; i++) { 45 | getline(&line, &len, fp); 46 | 47 | token = strtok(line, s); 48 | for (int j = 0; j < FEATURES; j++) { 49 | if (atoi(token) == 1) { 50 | int chunk_nr = j / INT_SIZE; 51 | int chunk_pos = j % INT_SIZE; 52 | X_train[i][chunk_nr] |= (1 << chunk_pos); 53 | } else { 54 | int chunk_nr = (j + FEATURES) / INT_SIZE; 55 | int chunk_pos = (j + FEATURES) % INT_SIZE; 56 | X_train[i][chunk_nr] |= (1 << chunk_pos); 57 | } 58 | token=strtok(NULL,s); 59 | } 60 | y_train[i] = atoi(token); 61 | } 62 | fclose(fp); 63 | 64 | // Test Dataset I 65 | 66 | for (int i = 0; i < NUMBER_OF_TEST_EXAMPLES; i++) { 67 | for (int j = 0; j < LA_CHUNKS; j++) { 68 | X_test[i][j] = 0; 69 | } 70 | } 71 | 72 | fp = fopen("MNISTTest.txt", "r"); 73 | if (fp == NULL) { 74 | printf("Error opening\n"); 75 | exit(EXIT_FAILURE); 76 | } 77 | 78 | for (int i = 0; i < NUMBER_OF_TEST_EXAMPLES; i++) { 79 | getline(&line, &len, fp); 80 | 81 | token = strtok(line, s); 82 | for (int j = 0; j < FEATURES; j++) { 83 | if (atoi(token) == 1) { 84 | int chunk_nr = j / INT_SIZE; 85 | int chunk_pos = j % INT_SIZE; 86 | X_test[i][chunk_nr] |= (1 << chunk_pos); 87 | } else { 88 | int chunk_nr = (j + FEATURES) / INT_SIZE; 89 | int chunk_pos = (j + FEATURES) % INT_SIZE; 90 | X_test[i][chunk_nr] |= (1 << chunk_pos); 91 | } 92 | token=strtok(NULL,s); 93 | } 94 | y_test[i] = atoi(token); 95 | } 96 | fclose(fp); 97 | 98 | // Sample of training dataset for speed 99 | 100 | for (int i = 0; i < NUMBER_OF_TEST_EXAMPLES; i++) { 101 | for (int j = 0; j < LA_CHUNKS; j++) { 102 | X_training_2[i][j] = 0; 103 | } 104 | } 105 | 106 | fp = fopen("MNISTTrainingSampled.txt", "r"); 107 | if (fp == NULL) { 108 | printf("Error opening\n"); 109 | exit(EXIT_FAILURE); 110 | } 111 | 112 | for (int i = 0; i < NUMBER_OF_TEST_EXAMPLES; i++) { 113 | getline(&line, &len, fp); 114 | 115 | token = strtok(line, s); 116 | for (int j = 0; j < FEATURES; j++) { 117 | if (atoi(token) == 1) { 118 | int chunk_nr = j / INT_SIZE; 119 | int chunk_pos = j % INT_SIZE; 120 | X_training_2[i][chunk_nr] |= (1 << chunk_pos); 121 | } else { 122 | int chunk_nr = (j + FEATURES) / INT_SIZE; 123 | int chunk_pos = (j + FEATURES) % INT_SIZE; 124 | X_training_2[i][chunk_nr] |= (1 << chunk_pos); 125 | } 126 | token=strtok(NULL,s); 127 | } 128 | y_training_2[i] = atoi(token); 129 | } 130 | fclose(fp); 131 | } 132 | 133 | void output_digit(unsigned int Xi[]) 134 | { 135 | for (int y = 0; y < 28; y++) { 136 | for (int x = 0; x < 28; x++) { 137 | int chunk_nr = (x + y*28) / INT_SIZE; 138 | int chunk_pos = (x + y*28) % INT_SIZE; 139 | 140 | if ((Xi[chunk_nr] & (1 << chunk_pos)) > 0) { 141 | printf("@"); 142 | } else { 143 | printf("."); 144 | } 145 | } 146 | printf("\n"); 147 | } 148 | } 149 | 150 | int main(void) 151 | { 152 | srand(time(NULL)); 153 | 154 | read_file(); 155 | 156 | int example = (int)(NUMBER_OF_TEST_EXAMPLES-1) * 1.0*rand()/RAND_MAX; 157 | printf("\nExample of Digit %d\n\n", y_test[example]); 158 | output_digit(X_test[example]); 159 | 160 | struct MultiClassTsetlinMachine *mc_tm = CreateMultiClassTsetlinMachine(); 161 | 162 | for (int i = 0; i < EPOCHS; i++) { 163 | printf("\nEPOCH %d\n", i+1); 164 | 165 | clock_t start_total = clock(); 166 | mc_tm_fit(mc_tm, X_train, y_train, NUMBER_OF_TRAINING_EXAMPLES, 1); 167 | clock_t end_total = clock(); 168 | double time_used = ((double) (end_total - start_total)) / CLOCKS_PER_SEC; 169 | 170 | printf("Training Time: %.1f s\n", time_used); 171 | 172 | start_total = clock(); 173 | float test_accuracy = mc_tm_evaluate(mc_tm, X_test, y_test, NUMBER_OF_TEST_EXAMPLES); 174 | end_total = clock(); 175 | time_used = ((double) (end_total - start_total)) / CLOCKS_PER_SEC; 176 | 177 | printf("Evaluation Time: %.1f s\n", time_used); 178 | printf("Test Accuracy: %.2f\n", 100*test_accuracy); 179 | 180 | float training_2_accuracy = mc_tm_evaluate(mc_tm, X_training_2, y_training_2, NUMBER_OF_TEST_EXAMPLES); 181 | printf("Training Accuracy: %.2f\n", 100*training_2_accuracy); 182 | } 183 | 184 | return 0; 185 | } 186 | -------------------------------------------------------------------------------- /MultiClassTsetlinMachineBits.c: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (c) 2019 Ole-Christoffer Granmo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | This code implements a multiclass version of the Tsetlin Machine from paper arXiv:1804.01508 24 | https://arxiv.org/abs/1804.01508 25 | 26 | */ 27 | 28 | #include 29 | #include 30 | 31 | #include "MultiClassTsetlinMachineBits.h" 32 | 33 | /**************************************/ 34 | /*** The Multiclass Tsetlin Machine ***/ 35 | /**************************************/ 36 | 37 | /*** Initialize Tsetlin Machine ***/ 38 | struct MultiClassTsetlinMachine *CreateMultiClassTsetlinMachine() 39 | { 40 | 41 | struct MultiClassTsetlinMachine *mc_tm; 42 | 43 | mc_tm = (void *)malloc(sizeof(struct MultiClassTsetlinMachine)); 44 | 45 | for (int i = 0; i < CLASSES; i++) { 46 | mc_tm->tsetlin_machines[i] = CreateTsetlinMachine(); 47 | } 48 | return mc_tm; 49 | } 50 | 51 | void mc_tm_initialize(struct MultiClassTsetlinMachine *mc_tm) 52 | { 53 | for (int i = 0; i < CLASSES; i++) { 54 | tm_initialize(mc_tm->tsetlin_machines[i]); 55 | } 56 | } 57 | 58 | /********************************************/ 59 | /*** Evaluate the Trained Tsetlin Machine ***/ 60 | /********************************************/ 61 | 62 | float mc_tm_evaluate(struct MultiClassTsetlinMachine *mc_tm, unsigned int X[][LA_CHUNKS], int y[], int number_of_examples) 63 | { 64 | int errors; 65 | int max_class; 66 | int max_class_sum; 67 | 68 | errors = 0; 69 | for (int l = 0; l < number_of_examples; l++) { 70 | /******************************************/ 71 | /*** Identify Class with Largest Output ***/ 72 | /******************************************/ 73 | 74 | max_class_sum = tm_score(mc_tm->tsetlin_machines[0], X[l]); 75 | max_class = 0; 76 | for (int i = 1; i < CLASSES; i++) { 77 | int class_sum = tm_score(mc_tm->tsetlin_machines[i], X[l]); 78 | if (max_class_sum < class_sum) { 79 | max_class_sum = class_sum; 80 | max_class = i; 81 | } 82 | } 83 | 84 | if (max_class != y[l]) { 85 | errors += 1; 86 | } 87 | } 88 | 89 | return 1.0 - 1.0 * errors / number_of_examples; 90 | } 91 | 92 | /******************************************/ 93 | /*** Online Training of Tsetlin Machine ***/ 94 | /******************************************/ 95 | 96 | // The Tsetlin Machine can be trained incrementally, one training example at a time. 97 | // Use this method directly for online and incremental training. 98 | 99 | void mc_tm_update(struct MultiClassTsetlinMachine *mc_tm, unsigned int Xi[], int target_class) 100 | { 101 | tm_update(mc_tm->tsetlin_machines[target_class], Xi, 1); 102 | 103 | // Randomly pick one of the other classes, for pairwise learning of class output 104 | unsigned int negative_target_class = (unsigned int)CLASSES * 1.0*rand()/((unsigned int)RAND_MAX + 1); 105 | while (negative_target_class == target_class) { 106 | negative_target_class = (unsigned int)CLASSES * 1.0*rand()/((unsigned int)RAND_MAX + 1); 107 | } 108 | tm_update(mc_tm->tsetlin_machines[negative_target_class], Xi, 0); 109 | } 110 | 111 | /**********************************************/ 112 | /*** Batch Mode Training of Tsetlin Machine ***/ 113 | /**********************************************/ 114 | 115 | void mc_tm_fit(struct MultiClassTsetlinMachine *mc_tm, unsigned int X[][LA_CHUNKS], int y[], int number_of_examples, int epochs) 116 | { 117 | for (int epoch = 0; epoch < epochs; epoch++) { 118 | // Add shuffling here... 119 | for (int i = 0; i < number_of_examples; i++) { 120 | mc_tm_update(mc_tm, X[i], y[i]); 121 | } 122 | } 123 | } 124 | 125 | -------------------------------------------------------------------------------- /MultiClassTsetlinMachineBits.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (c) 2019 Ole-Christoffer Granmo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | This code implements a multiclass version of the Tsetlin Machine from paper arXiv:1804.01508 24 | https://arxiv.org/abs/1804.01508 25 | 26 | */ 27 | 28 | #include "TsetlinMachineBits.h" 29 | #include "MultiClassTsetlinMachineConfig.h" 30 | 31 | struct MultiClassTsetlinMachine { 32 | struct TsetlinMachine *tsetlin_machines[CLASSES]; 33 | }; 34 | 35 | struct MultiClassTsetlinMachine *CreateMultiClassTsetlinMachine(); 36 | 37 | void mc_tm_initialize(struct MultiClassTsetlinMachine *mc_tm); 38 | 39 | void mc_tm_initialize_random_streams(struct MultiClassTsetlinMachine *mc_tm); 40 | 41 | float mc_tm_evaluate(struct MultiClassTsetlinMachine *mc_tm, unsigned int X[][LA_CHUNKS], int y[], int number_of_examples); 42 | 43 | void mc_tm_fit(struct MultiClassTsetlinMachine *mc_tm, unsigned int X[][LA_CHUNKS], int y[], int number_of_examples, int epochs); 44 | -------------------------------------------------------------------------------- /MultiClassTsetlinMachineConfig.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (c) 2019 Ole-Christoffer Granmo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | This code implements a multiclass version of the Tsetlin Machine from paper arXiv:1804.01508 24 | https://arxiv.org/abs/1804.01508 25 | 26 | */ 27 | 28 | #define CLASSES 10 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A Fast Tsetlin Machine Implementation Employing Bit-Wise Operators 2 | An implementation of the Tsetlin Machine (https://arxiv.org/abs/1804.01508) using bitwise operations for increased learning- and classification speed. 3 | 4 | On the MNIST dataset, the bit manipulation leads to approx. 5 | * 10 times smaller memory footprint, 6 | * 8 times quicker classification, and 7 | * 10 times faster learning, 8 | 9 | compared to the vanilla Cython (https://github.com/cair/TsetlinMachine) and C (https://github.com/cair/TsetlinMachineC) implementations. 10 | 11 | ## Bit-Based Representation and Manipulation of Patterns 12 | 13 | The Tsetlin Machine solves complex pattern recognition problems with propositional formulas, composed by a collective of Tsetlin Automata. In this implementation, we express both inputs, patterns, and outputs as bits, while recognition and learning rely on bit manipulation. Briefly stated, the states of the Tsetlin Automata are jointly represented using multiple sequences of bits (e.g., 8 sequences to represent an 8 bit state index). Sequence 1 contains the first bit of each state index. Sequence 2 contains the second bit, and so on, as exemplified below for 24 Tsetlin Automata: 14 | 15 | ![Figure 4](https://github.com/olegranmo/blob/blob/master/Bit_Manipulation_3.png) 16 | 17 | The benefit of this representation is that the action of each Tsetlin Automaton is readily available from the most significant bit (sequence 8 in the figure). Thus, the output (recognized or not recognized pattern) can be obtained from the input based on fast bitwise operators (NOT, AND, and CMP - comparison). When deployed after training, only the sequence containing the most significant bit is required. The other sequences can be discarded because these bits are only used to keep track of the learning. This provides a further reduction in memory usage. 18 | 19 | ## MNIST Demo 20 | ```bash 21 | unzip BinarizedMNISTData.zip 22 | make 23 | ./MNISTDemoBits 24 | 25 | EPOCH 1 26 | Training Time: 33.1 s 27 | Evaluation Time: 13.3 s 28 | Test Accuracy: 94.42 29 | Training Accuracy: 95.99 30 | 31 | EPOCH 2 32 | Training Time: 25.8 s 33 | Evaluation Time: 13.8 s 34 | Test Accuracy: 95.39 35 | Training Accuracy: 96.72 36 | 37 | EPOCH 3 38 | Training Time: 24.6 s 39 | Evaluation Time: 13.9 s 40 | Test Accuracy: 95.77 41 | Training Accuracy: 97.21 42 | 43 | EPOCH 4 44 | Training Time: 23.9 s 45 | Evaluation Time: 13.9 s 46 | Test Accuracy: 96.27 47 | Training Accuracy: 97.54 48 | 49 | EPOCH 5 50 | Training Time: 23.3 s 51 | Evaluation Time: 14.2 s 52 | Test Accuracy: 96.52 53 | Training Accuracy: 97.76 54 | ... 55 | 56 | EPOCH 396 57 | Training Time: 18.5 s 58 | Evaluation Time: 15.0 s 59 | Test Accuracy: 98.15 60 | Training Accuracy: 99.90 61 | 62 | EPOCH 397 63 | Training Time: 18.3 s 64 | Evaluation Time: 14.5 s 65 | Test Accuracy: 98.12 66 | Training Accuracy: 99.91 67 | 68 | EPOCH 398 69 | Training Time: 18.2 s 70 | Evaluation Time: 14.4 s 71 | Test Accuracy: 98.13 72 | Training Accuracy: 99.92 73 | 74 | EPOCH 399 75 | Training Time: 18.4 s 76 | Evaluation Time: 14.9 s 77 | Test Accuracy: 98.19 78 | Training Accuracy: 99.90 79 | 80 | EPOCH 400 81 | Training Time: 18.0 s 82 | Evaluation Time: 14.4 s 83 | Test Accuracy: 98.12 84 | Training Accuracy: 99.90 85 | ``` 86 | ## Data Preparation 87 | 88 | The included dataset is a binarized, but otherwise unenhanced, version of the MNIST dataset (http://yann.lecun.com/exdb/mnist/), downloaded from https://github.com/mnielsen/neural-networks-and-deep-learning/tree/master/data. The complete dataset was binarized by replacing pixel values larger than 0.3 with 1 (with the original pixel values ranging from 0 to 1). Pixel values below or equal to 0.3 were replaced with 0. The following image is an example of the images produced. 89 | 90 | ```bash 91 | ............................ 92 | ............................ 93 | ............................ 94 | ............................ 95 | ............................ 96 | ............................ 97 | ............@@.............. 98 | ...........@@.....@@@....... 99 | ...........@@.....@@........ 100 | ..........@@......@@........ 101 | .........@@@......@@........ 102 | .........@@.......@@........ 103 | .........@@......@@@........ 104 | ........@@@.@@@..@@@........ 105 | ........@@@@@@@@@@@@........ 106 | .......@@@@@@@@@@@@......... 107 | .......@@........@@......... 108 | .................@@......... 109 | ................@@@......... 110 | ................@@@......... 111 | ................@@@......... 112 | ................@@@......... 113 | ................@@@......... 114 | ................@@@......... 115 | ................@@@......... 116 | ................@@@......... 117 | ............................ 118 | ............................ 119 | ``` 120 | ## Learning Behaviour 121 | The below figure depicts average learning progress (across 50 runs) of the Tsetlin Machine on the included MNIST dataset. 122 | 123 | ![Figure 4](https://github.com/olegranmo/blob/blob/master/learning_progress.png) 124 | 125 | As seen in the figure, both test and training accuracy increase almost monotonically across the epochs. Even while accuracy on the training data approaches 99.9%, accuracy on the test data continues to increase as well, hitting 98.2% after 400 epochs. This is quite different from what occurs with backpropagation on a neural network, where accuracy on test data starts to drop at some point due to overfitting, without proper regularization mechanisms. 126 | 127 | ## Further Work 128 | 129 | * Perform a more extensive hyperparameter search (manipulating THRESHOLD, CLAUSES, STATE_BITS, and S in TsetlinMachineBitsConfig.h). 130 | * Evaluate different binarization and data augmentation approaches for MNIST, including deskewing, noise removal, blurring, and pixel shifts. 131 | * Investigate effect of using an ensemble of Tsetlin Machines. 132 | * Optimize code base further. 133 | 134 | ## Citation 135 | 136 | ```bash 137 | @article{granmo2018tsetlin, 138 | author = {{Granmo}, Ole-Christoffer}, 139 | title = "{The Tsetlin Machine - A Game Theoretic Bandit Driven Approach to Optimal Pattern Recognition with Propositional Logic}", 140 | journal={arXiv preprint arXiv:1804.01508}, 141 | year={2018} 142 | } 143 | ``` 144 | 145 | ## Licence 146 | 147 | Copyright (c) 2019 Ole-Christoffer Granmo 148 | 149 | Permission is hereby granted, free of charge, to any person obtaining a copy 150 | of this software and associated documentation files (the "Software"), to deal 151 | in the Software without restriction, including without limitation the rights 152 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 153 | copies of the Software, and to permit persons to whom the Software is 154 | furnished to do so, subject to the following conditions: 155 | 156 | The above copyright notice and this permission notice shall be included in all 157 | copies or substantial portions of the Software. 158 | 159 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 160 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 161 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 162 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 163 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 164 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 165 | SOFTWARE. 166 | 167 | -------------------------------------------------------------------------------- /TsetlinMachineBits.c: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (c) 2019 Ole-Christoffer Granmo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | This code implements a multiclass version of the Tsetlin Machine from paper arXiv:1804.01508 24 | https://arxiv.org/abs/1804.01508 25 | 26 | */ 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #include "fast_rand.h" 35 | 36 | #include "TsetlinMachineBits.h" 37 | 38 | struct TsetlinMachine *CreateTsetlinMachine() 39 | { 40 | /* Set up the Tsetlin Machine structure */ 41 | 42 | struct TsetlinMachine *tm = (void *)malloc(sizeof(struct TsetlinMachine)); 43 | 44 | tm_initialize(tm); 45 | 46 | return tm; 47 | } 48 | 49 | void tm_initialize(struct TsetlinMachine *tm) 50 | { 51 | /* Set up the Tsetlin Machine structure */ 52 | 53 | for (int j = 0; j < CLAUSES; ++j) { 54 | for (int k = 0; k < LA_CHUNKS; ++k) { 55 | for (int b = 0; b < STATE_BITS-1; ++b) { 56 | (*tm).ta_state[j][k][b] = ~0; 57 | } 58 | (*tm).ta_state[j][k][STATE_BITS-1] = 0; 59 | } 60 | } 61 | } 62 | 63 | static inline void tm_initialize_random_streams(struct TsetlinMachine *tm) 64 | { 65 | // Initialize all bits to zero 66 | memset((*tm).feedback_to_la, 0, LA_CHUNKS*sizeof(unsigned int)); 67 | 68 | int n = 2 * FEATURES; 69 | double p = 1.0 / S; 70 | int active = normal(n * p, n * p * (1 - p)); 71 | active = active >= n ? n : active; 72 | active = active < 0 ? 0 : active; 73 | while (active--) { 74 | int f = fast_rand() % (2 * FEATURES); 75 | while ((*tm).feedback_to_la[f / INT_SIZE] & (1 << (f % INT_SIZE))) { 76 | f = fast_rand() % (2 * FEATURES); 77 | } 78 | (*tm).feedback_to_la[f / INT_SIZE] |= 1 << (f % INT_SIZE); 79 | } 80 | } 81 | 82 | // Increment the states of each of those 32 Tsetlin Automata flagged in the active bit vector. 83 | static inline void tm_inc(struct TsetlinMachine *tm, int clause, int chunk, unsigned int active) 84 | { 85 | unsigned int carry, carry_next; 86 | 87 | carry = active; 88 | for (int b = 0; b < STATE_BITS; ++b) { 89 | if (carry == 0) 90 | break; 91 | 92 | carry_next = (*tm).ta_state[clause][chunk][b] & carry; // Sets carry bits (overflow) passing on to next bit 93 | (*tm).ta_state[clause][chunk][b] = (*tm).ta_state[clause][chunk][b] ^ carry; // Performs increments with XOR 94 | carry = carry_next; 95 | } 96 | 97 | if (carry > 0) { 98 | for (int b = 0; b < STATE_BITS; ++b) { 99 | (*tm).ta_state[clause][chunk][b] |= carry; 100 | } 101 | } 102 | } 103 | 104 | // Decrement the states of each of those 32 Tsetlin Automata flagged in the active bit vector. 105 | static inline void tm_dec(struct TsetlinMachine *tm, int clause, int chunk, unsigned int active) 106 | { 107 | unsigned int carry, carry_next; 108 | 109 | carry = active; 110 | for (int b = 0; b < STATE_BITS; ++b) { 111 | if (carry == 0) 112 | break; 113 | 114 | carry_next = (~(*tm).ta_state[clause][chunk][b]) & carry; // Sets carry bits (overflow) passing on to next bit 115 | (*tm).ta_state[clause][chunk][b] = (*tm).ta_state[clause][chunk][b] ^ carry; // Performs increments with XOR 116 | carry = carry_next; 117 | } 118 | 119 | if (carry > 0) { 120 | for (int b = 0; b < STATE_BITS; ++b) { 121 | (*tm).ta_state[clause][chunk][b] &= ~carry; 122 | } 123 | } 124 | } 125 | 126 | /* Sum up the votes for each class */ 127 | static inline int sum_up_class_votes(struct TsetlinMachine *tm) 128 | { 129 | int class_sum = 0; 130 | 131 | for (int j = 0; j < CLAUSE_CHUNKS; j++) { 132 | class_sum += __builtin_popcount((*tm).clause_output[j] & 0x55555555); // 0101 133 | class_sum -= __builtin_popcount((*tm).clause_output[j] & 0xaaaaaaaa); // 1010 134 | } 135 | 136 | class_sum = (class_sum > THRESHOLD) ? THRESHOLD : class_sum; 137 | class_sum = (class_sum < -THRESHOLD) ? -THRESHOLD : class_sum; 138 | 139 | return class_sum; 140 | } 141 | 142 | /* Calculate the output of each clause using the actions of each Tsetline Automaton. */ 143 | static inline void tm_calculate_clause_output(struct TsetlinMachine *tm, unsigned int Xi[], int predict) 144 | { 145 | memset((*tm).clause_output, 0, CLAUSE_CHUNKS*sizeof(unsigned int)); 146 | 147 | for (int j = 0; j < CLAUSES; j++) { 148 | unsigned int output = 1; 149 | unsigned int all_exclude = 1; 150 | for (int k = 0; k < LA_CHUNKS-1; k++) { 151 | output = output && ((*tm).ta_state[j][k][STATE_BITS-1] & Xi[k]) == (*tm).ta_state[j][k][STATE_BITS-1]; 152 | 153 | if (!output) { 154 | break; 155 | } 156 | all_exclude = all_exclude && ((*tm).ta_state[j][k][STATE_BITS-1] == 0); 157 | } 158 | 159 | output = output && 160 | ((*tm).ta_state[j][LA_CHUNKS-1][STATE_BITS-1] & Xi[LA_CHUNKS-1] & FILTER) == 161 | ((*tm).ta_state[j][LA_CHUNKS-1][STATE_BITS-1] & FILTER); 162 | 163 | all_exclude = all_exclude && (((*tm).ta_state[j][LA_CHUNKS-1][STATE_BITS-1] & FILTER) == 0); 164 | 165 | output = output && !(predict == PREDICT && all_exclude == 1); 166 | 167 | if (output) { 168 | unsigned int clause_chunk = j / INT_SIZE; 169 | unsigned int clause_chunk_pos = j % INT_SIZE; 170 | 171 | (*tm).clause_output[clause_chunk] |= (1 << clause_chunk_pos); 172 | } 173 | } 174 | } 175 | 176 | /******************************************/ 177 | /*** Online Training of Tsetlin Machine ***/ 178 | /******************************************/ 179 | 180 | // The Tsetlin Machine can be trained incrementally, one training example at a time. 181 | // Use this method directly for online and incremental training. 182 | 183 | void tm_update(struct TsetlinMachine *tm, unsigned int Xi[], int target) 184 | { 185 | /*******************************/ 186 | /*** Calculate Clause Output ***/ 187 | /*******************************/ 188 | 189 | tm_calculate_clause_output(tm, Xi, UPDATE); 190 | 191 | /***************************/ 192 | /*** Sum up Clause Votes ***/ 193 | /***************************/ 194 | 195 | int class_sum = sum_up_class_votes(tm); 196 | 197 | /*********************************/ 198 | /*** Train Individual Automata ***/ 199 | /*********************************/ 200 | 201 | // Calculate feedback to clauses 202 | 203 | float p = (1.0/(THRESHOLD*2))*(THRESHOLD + (1 - 2*target)*class_sum); 204 | memset((*tm).feedback_to_clauses, 0, CLAUSE_CHUNKS*sizeof(int)); 205 | for (int j = 0; j < CLAUSES; j++) { 206 | unsigned int clause_chunk = j / INT_SIZE; 207 | unsigned int clause_chunk_pos = j % INT_SIZE; 208 | 209 | (*tm).feedback_to_clauses[clause_chunk] |= (((float)fast_rand())/((float)FAST_RAND_MAX) <= p) << clause_chunk_pos; 210 | } 211 | 212 | for (int j = 0; j < CLAUSES; j++) { 213 | unsigned int clause_chunk = j / INT_SIZE; 214 | unsigned int clause_chunk_pos = j % INT_SIZE; 215 | 216 | if (!((*tm).feedback_to_clauses[clause_chunk] & (1 << clause_chunk_pos))) { 217 | continue; 218 | } 219 | 220 | if ((2*target-1) * (1 - 2 * (j & 1)) == -1) { 221 | if (((*tm).clause_output[clause_chunk] & (1 << clause_chunk_pos)) > 0) { 222 | // Type II Feedback 223 | 224 | for (int k = 0; k < LA_CHUNKS; ++k) { 225 | tm_inc(tm, j, k, (~Xi[k]) & (~(*tm).ta_state[j][k][STATE_BITS-1])); 226 | } 227 | } 228 | } else if ((2*target-1) * (1 - 2 * (j & 1)) == 1) { 229 | // Type I Feedback 230 | 231 | tm_initialize_random_streams(tm); 232 | 233 | if (((*tm).clause_output[clause_chunk] & (1 << clause_chunk_pos)) > 0) { 234 | for (int k = 0; k < LA_CHUNKS; ++k) { 235 | #ifdef BOOST_TRUE_POSITIVE_FEEDBACK 236 | tm_inc(tm, j, k, Xi[k]); 237 | #else 238 | tm_inc(tm, j, k, Xi[k] & (~tm->feedback_to_la[k])); 239 | #endif 240 | 241 | tm_dec(tm, j, k, (~Xi[k]) & tm->feedback_to_la[k]); 242 | } 243 | } else { 244 | for (int k = 0; k < LA_CHUNKS; ++k) { 245 | tm_dec(tm, j, k, tm->feedback_to_la[k]); 246 | } 247 | } 248 | } 249 | } 250 | } 251 | 252 | int tm_score(struct TsetlinMachine *tm, unsigned int Xi[]) { 253 | /*******************************/ 254 | /*** Calculate Clause Output ***/ 255 | /*******************************/ 256 | 257 | tm_calculate_clause_output(tm, Xi, PREDICT); 258 | 259 | /***************************/ 260 | /*** Sum up Clause Votes ***/ 261 | /***************************/ 262 | 263 | return sum_up_class_votes(tm); 264 | } 265 | 266 | int tm_get_state(struct TsetlinMachine *tm, int clause, int la) 267 | { 268 | int la_chunk = la / INT_SIZE; 269 | int chunk_pos = la % INT_SIZE; 270 | 271 | int state = 0; 272 | for (int b = 0; b < STATE_BITS; ++b) { 273 | if ((*tm).ta_state[clause][la_chunk][b] & (1 << chunk_pos)) { 274 | state |= 1 << b; 275 | } 276 | } 277 | 278 | return state; 279 | } 280 | 281 | int tm_action(struct TsetlinMachine *tm, int clause, int la) 282 | { 283 | int la_chunk = la / INT_SIZE; 284 | int chunk_pos = la % INT_SIZE; 285 | 286 | return ((*tm).ta_state[clause][la_chunk][STATE_BITS-1] & (1 << chunk_pos)) > 0; 287 | } 288 | 289 | 290 | -------------------------------------------------------------------------------- /TsetlinMachineBits.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (c) 2019 Ole-Christoffer Granmo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | This code implements a multiclass version of the Tsetlin Machine from paper arXiv:1804.01508 24 | https://arxiv.org/abs/1804.01508 25 | 26 | */ 27 | 28 | #include "TsetlinMachineBitsConfig.h" 29 | 30 | #define INT_SIZE 32 31 | 32 | #define LA_CHUNKS (((2*FEATURES-1)/INT_SIZE + 1)) 33 | #define CLAUSE_CHUNKS ((CLAUSES-1)/INT_SIZE + 1) 34 | 35 | #if ((FEATURES*2) % 32 != 0) 36 | #define FILTER (~(0xffffffff << ((FEATURES*2) % INT_SIZE))) 37 | #else 38 | #define FILTER 0xffffffff 39 | #endif 40 | 41 | #define PREDICT 1 42 | #define UPDATE 0 43 | 44 | struct TsetlinMachine { 45 | unsigned int ta_state[CLAUSES][LA_CHUNKS][STATE_BITS]; 46 | unsigned int clause_output[CLAUSE_CHUNKS]; 47 | unsigned int feedback_to_la[LA_CHUNKS]; 48 | int feedback_to_clauses[CLAUSE_CHUNKS]; 49 | }; 50 | 51 | struct TsetlinMachine *CreateTsetlinMachine(); 52 | 53 | void tm_initialize(struct TsetlinMachine *tm); 54 | 55 | void tm_update(struct TsetlinMachine *tm, unsigned int Xi[], int target); 56 | 57 | int tm_score(struct TsetlinMachine *tm, unsigned int Xi[]); 58 | 59 | int tm_get_state(struct TsetlinMachine *tm, int clause, int la); 60 | 61 | int tm_action(struct TsetlinMachine *tm, int clause, int la); 62 | -------------------------------------------------------------------------------- /TsetlinMachineBitsConfig.h: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright (c) 2019 Ole-Christoffer Granmo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | This code implements a multiclass version of the Tsetlin Machine from paper arXiv:1804.01508 24 | https://arxiv.org/abs/1804.01508 25 | 26 | */ 27 | 28 | #define THRESHOLD 50 29 | #define CLAUSES 2000 30 | #define S 10.0 31 | 32 | #define STATE_BITS 8 33 | #define BOOST_TRUE_POSITIVE_FEEDBACK 1 34 | #define FEATURES 784 35 | -------------------------------------------------------------------------------- /fast_rand.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by Yasser Phoulady on 2019-04-01. 3 | // 4 | 5 | #ifndef C_BITWISE_TSETLIN_MACHINE_FAST_RAND_H 6 | #define C_BITWISE_TSETLIN_MACHINE_FAST_RAND_H 7 | 8 | #include 9 | 10 | static uint64_t const multiplier = 6364136223846793005u; 11 | static uint64_t mcg_state = 0xcafef00dd15ea5e5u; 12 | 13 | inline static uint32_t pcg32_fast() { 14 | uint64_t x = mcg_state; 15 | unsigned int count = (unsigned int) (x >> 61); // 61 = 64 - 3 16 | 17 | mcg_state = x * multiplier; 18 | return (uint32_t) ((x ^ x >> 22) >> (22 + count)); // 22 = 32 - 3 - 7 19 | } 20 | 21 | //static void pcg32_fast_init(uint64_t seed) { 22 | // mcg_state = 2 * seed + 1; 23 | // pcg32_fast(); 24 | //} 25 | 26 | #define FAST_RAND_MAX UINT32_MAX 27 | #define fast_rand() pcg32_fast() 28 | 29 | // Box–Muller transform 30 | inline static int normal(double mean, double variance) { 31 | double u1 = (double) (fast_rand() + 1) / ((double) FAST_RAND_MAX + 1), u2 = (double) fast_rand() / FAST_RAND_MAX; // u1 in (0, 1] and u2 in [0, 1] 32 | double n1 = sqrt(-2 * log(u1)) * sin(8 * atan(1) * u2); 33 | return (int) round(mean + sqrt(variance) * n1); 34 | } 35 | 36 | inline static int binomial(int n, double p) { 37 | return normal(n * p, n * p * (1 - p)); 38 | } 39 | 40 | // Knuth's random Poisson-distributed number 41 | inline static int poisson(double lambda) { 42 | int k = 0; 43 | double l = exp(-lambda), p = 1; 44 | while (p > l) { 45 | ++k; 46 | p *= (double) fast_rand() / FAST_RAND_MAX; 47 | } 48 | return k - 1; 49 | } 50 | 51 | #endif //C_BITWISE_TSETLIN_MACHINE_FAST_RAND_H 52 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | MNISTDemoBits: MultiClassTsetlinMachineBits.c MultiClassTsetlinMachineBits.h TsetlinMachineBits.c TsetlinMachineBitsConfig.h TsetlinMachineBits.h MNISTDemoBits.c 2 | gcc -Wall -O3 -ffast-math -o MNISTDemoBits MNISTDemoBits.c MultiClassTsetlinMachineBits.c TsetlinMachineBits.c -lm 3 | 4 | clean: 5 | rm *.o MNISTDemoBits 6 | --------------------------------------------------------------------------------