├── README.md ├── doc ├── STA0.png ├── STA1.png ├── STA1fdp.png ├── STA2.png ├── STA3.1.png ├── STA3.png ├── STA5.png ├── Text compression.K.Orav.2023.pdf ├── fchar.article1.png └── sta.txt ├── english.dic ├── for cmix ├── fxcmv1.cpp ├── fxcmv1.h └── readme.txt ├── fxcm.cpp └── tools ├── dictionary.dic └── states.cpp /README.md: -------------------------------------------------------------------------------- 1 | # fxcm 2 | 3 | Is a experimental file compressor for text and is based on [PAQ](http://www.mattmahoney.net/dc/text.html#1250) architecture. Designed to replace paq8hp model in [cmix-hp](https://github.com/byronknoll/cmix-hp). 4 | 5 | Dictionary encode/decode [DIC](https://github.com/kaitz/fxd). Wikipedia transform [WIT](https://github.com/kaitz/wit2). 6 | 7 | | |File|Transforms|Size|Compressed|Time sec|Memory MB 8 | | --- | --- | --- | --- | --- | --- | --- | 9 | |fxcm v1|enwik8|-|100000000|16675996|2934|1840| 10 | |fxcm v1|enwik9|-|1000000000|135192577|26322|1840| 11 | |fxcm v2|enwik8|WIT,DIC|57088865|15761972|1880|1840| 12 | |fxcm v2|enwik9|WIT,DIC|593869820|126234551|17121|1840| 13 | |fxcm v3|enwik8|-|100000000|16648562| 2780 |1829 | 14 | |fxcm v3|enwik9|-|1000000000|134963229| | | 15 | |fxcm v4|enwik8|WIT,DIC|57088865|15746213| | | 16 | |fxcm v4|enwik9|WIT,DIC|593869820|126104581|18875 | 1829| 17 | |fxcm v5|enwik8|-|100000000|16638087|3507|1829| 18 | |fxcm v5|enwik9|-| 1000000000|134868134|29813| 1829| 19 | |fxcm v6|enwik8|WIT,DIC| 57088865|15730202|2168 |1829| 20 | |fxcm v6|enwik9|WIT,DIC|593869820|125982772|19843| 1829| 21 | |fxcm v8|enwik8|WIT,DIC|57088865|15725558|1890|1829| 22 | |fxcm v8|enwik9|WIT,DIC|593869820|125944742|17364 | 1829| 23 | |fxcm v9|enwik8|-|100000000|16598942|2899|1834| 24 | |fxcm v9|enwik9|-|1000000000|134493830|28183| 1834| 25 | |fxcm v10|enwik8|WIT,DIC|57088865|15706823|1758| 1834| 26 | |fxcm v10|enwik9|WIT,DIC|593869820|125737089|18948| 1834| 27 | |fxcm v11|enwik8|-|100000000|16565161|3550|1834| 28 | |fxcm v11|enwik9|-|1000000000|134067705|28993| 1834| 29 | |fxcm v12|enwik8|WIT,DIC|57088865|15687987|1988| 1834| 30 | |fxcm v12|enwik9|WIT,DIC|593869820|125586708|20036| 1834| 31 | |fxcm v14|enwik9|WIT,DIC|593869820|125483355|16174| 1834| 32 | |fxcm v15|enwik9|-|1000000000|133766633|28995| 1834| 33 | |fxcm v16|enwik9|WIT,DIC|593869820|125314546|16350| 1834| 34 | |fxcm v24|enwik8|WIT,DIC|57121234*|15395484|2922| 4482| 35 | |fxcm v24|enwik9|WIT,DIC|594026973*|121823599|27055| 4482| 36 | |paq8pxd_v107 -s7|enwik8|DIC-(internal)|100000000|16408142|11189|1460| 37 | |paq8pxd_v107 -s8|enwik8|DIC-(internal)|100000000|16182108|11473|2264| 38 | |paq8n -8|enwik8|-|100000000|17916450|5663|1567| 39 | |paq8n -8|enwik8|WIT,DIC|57088865|16905680|3457|1567| 40 | |paq8o10t -8|enwik8||100000000|17772821|6017|1517| 41 | |paq8o10t -8|enwik8|WIT,DIC|57088865|17101300|2914|1517| 42 | |paq8px_v208fix1 -8|enwik8||100000000|16499082|23498|2163| 43 | |paq8px_v208fix1 -8|enwik8|WIT,DIC|57088865|15969942|14087|2163| 44 | |paq8hp12any|enwik8|WIT,DIC|57088865|16131394|2393|1813| 45 | |paq8hp12any|enwik9|WIT,DIC|593869820|130573629|24396|1813| 46 | 47 | *Input was parsed with updated DIC. 48 | 49 | Test computer parameters: Intel Core i5 4460 3.2GHz, DDR3 PC3-12800 (800MHz) 32GB. 50 | 51 | fxcm v1,3,.. is for enwi8, enwik9 or any wiki dump. 52 | 53 | ​fxcm v2,4,.. is when wiki dump is preprocessed. 54 | 55 | Results: https://docs.google.com/document/d/1AGx--3zFy6raMNm5diAxk6GPBUJrxLgaxM9LTsMF1RE/ 56 | 57 | https://encode.su/threads/4116-fxcm 58 | -------------------------------------------------------------------------------- /doc/STA0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaitz/fxcm/739118ba1fe995934397c09556ca3ee477521dc6/doc/STA0.png -------------------------------------------------------------------------------- /doc/STA1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaitz/fxcm/739118ba1fe995934397c09556ca3ee477521dc6/doc/STA1.png -------------------------------------------------------------------------------- /doc/STA1fdp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaitz/fxcm/739118ba1fe995934397c09556ca3ee477521dc6/doc/STA1fdp.png -------------------------------------------------------------------------------- /doc/STA2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaitz/fxcm/739118ba1fe995934397c09556ca3ee477521dc6/doc/STA2.png -------------------------------------------------------------------------------- /doc/STA3.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaitz/fxcm/739118ba1fe995934397c09556ca3ee477521dc6/doc/STA3.1.png -------------------------------------------------------------------------------- /doc/STA3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaitz/fxcm/739118ba1fe995934397c09556ca3ee477521dc6/doc/STA3.png -------------------------------------------------------------------------------- /doc/STA5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaitz/fxcm/739118ba1fe995934397c09556ca3ee477521dc6/doc/STA5.png -------------------------------------------------------------------------------- /doc/Text compression.K.Orav.2023.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaitz/fxcm/739118ba1fe995934397c09556ca3ee477521dc6/doc/Text compression.K.Orav.2023.pdf -------------------------------------------------------------------------------- /doc/fchar.article1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaitz/fxcm/739118ba1fe995934397c09556ca3ee477521dc6/doc/fchar.article1.png -------------------------------------------------------------------------------- /doc/sta.txt: -------------------------------------------------------------------------------- 1 | STAx are stateables generated with graphviz. 2 | 3 | https://graphviz.org/Gallery/directed/fsm.html -------------------------------------------------------------------------------- /for cmix/fxcmv1.cpp: -------------------------------------------------------------------------------- 1 | // This is adapted from fxcm 2 | 3 | /* 4 | Copyright (C) 2023 Kaido Orav 5 | 6 | LICENSE 7 | 8 | This program is free software; you can redistribute it and/or 9 | modify it under the terms of the GNU General Public License as 10 | published by the Free Software Foundation; either version 2 of 11 | the License, or (at your option) any later version. 12 | 13 | This program is distributed in the hope that it will be useful, but 14 | WITHOUT ANY WARRANTY; without even the implied warranty of 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 | General Public License for more details at 17 | Visit . 18 | */ 19 | 20 | #include "fxcmv1.h" 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | //#include 32 | #define NDEBUG 33 | 34 | #ifndef DEFAULT_OPTION 35 | #define DEFAULT_OPTION 8 36 | #endif 37 | 38 | #ifndef NOASM 39 | #define NOASM 40 | #endif 41 | 42 | namespace fxcmv1 { 43 | 44 | typedef unsigned char U8; 45 | typedef unsigned short U16; 46 | typedef unsigned int U32; 47 | 48 | #ifndef min 49 | inline int min(int a, int b) {return a model_predictions(0.5, exported_models + num_extra_predictions); 57 | unsigned int prediction_index = 0; 58 | float conversion_factor = 1.0 / 4095; 59 | 60 | void AddPrediction(int x) { 61 | model_predictions[prediction_index++] = x * conversion_factor; 62 | } 63 | 64 | void ResetPredictions() { 65 | prediction_index = 0; 66 | } 67 | 68 | 69 | 70 | //#define TEXTMODE // comment this to get version 8 for dictionary proccessed input (ex. drt, paq8hp -0, cmix -c) 71 | 72 | #ifdef TEXTMODE 73 | #define VERSION 15 74 | #else 75 | #define VERSION 16 76 | #endif 77 | 78 | 79 | #include 80 | #include 81 | //#include 82 | #define NDEBUG // remove for debugging (turns on Array bound checks) 83 | #include 84 | 85 | #ifdef UNIX // not tested 86 | #include 87 | #include 88 | #include 89 | #include 90 | #include 91 | #include 92 | #include 93 | #include 94 | #include 95 | #endif 96 | 97 | // 8, 16, 32 bit unsigned types (adjust as appropriate) 98 | typedef unsigned char U8; 99 | typedef unsigned short U16; 100 | typedef unsigned int U32; 101 | typedef unsigned long long int U64; 102 | 103 | 104 | // min, max functions 105 | #if !defined(WINDOWS) || !defined (min) 106 | //inline int min(int a, int b) {return a 112 | //#include 113 | // print message if any, and exit 114 | /*void quit(const char* message=0) { 115 | #ifdef ERRMSG 116 | printf("%s",message); 117 | #endif 118 | exit(1); 119 | }*/ 120 | //////////////////////////// Array //////////////////////////// 121 | 122 | template void alloc(T*&ptr, int c) { 123 | ptr=(T*)calloc(c, sizeof(T)); 124 | if (!ptr) exit(1);//quit("Out of memory.\n"); 125 | } 126 | 127 | // for aligned data 128 | template void alloc1(T*&data, int c,T*&ptr,const int align=16) { 129 | ptr=(T*)calloc(c, sizeof(T)); 130 | if (!ptr) exit(1);//quit("Out of memory.\n"); 131 | data=(T*)(((uintptr_t)ptr+(align-1)) & ~(uintptr_t)(align-1)); 132 | 133 | } 134 | 135 | // Squash returns p = 1/(1 + exp(-d)), d scaled by 8 bits, p scaled by 12 bits 136 | short sqt[4095]; 137 | 138 | int squashc(int d ) { 139 | if (d < -2047)return 1; 140 | if (d > 2047)return 4095; 141 | float p = 1.0f / (1.0f + exp(-d / 256.0)); 142 | p *= 4096.0; 143 | U32 pi = (U32)round(p); 144 | if (pi > 4095)pi = 4095; 145 | if (pi < 1)pi = 1; 146 | return pi; 147 | } 148 | 149 | inline int squash(int d) { 150 | if (d < -2047)return 1; 151 | if (d > 2047)return 4095; 152 | return sqt[d + 2047]; 153 | } 154 | 155 | // Stretch is inverse of squash. d = ln(p/(1-p)), d scaled by 8 bits, p by 12 bits. 156 | // d has range -2047 to 2047 representing -8 to 8. p has range 0 to 4095. 157 | short strt[4096]; 158 | 159 | int stretchc(int p) { 160 | assert(p >= 0 && p <= 4095); 161 | if (p == 0)p = 1; 162 | float f = p / 4096.0f; 163 | float d = log(f / (1.0f - f)) * 256.0f; 164 | int di = (int)round(d); 165 | if (di > 2047)di = 2047; 166 | if (di < -2047)di = -2047; 167 | return di; 168 | } 169 | 170 | inline short stretch(int p) { 171 | return strt[p]; 172 | } 173 | struct BlockData { 174 | int y; // Last bit, 0 or 1, set by encoder 175 | int c0; // Last 0-7 bits of the partial byte with a leading 1 bit (1-255) 176 | U32 c4; // Last 4 whole bytes, packed. 177 | int bpos; // bits in c0 (0 to 7) 178 | int blpos; // Relative position in block 179 | int bposshift; 180 | int c0shift_bpos; 181 | struct Inputs{ 182 | int ncount; // mixer input count 183 | short *n,*ptr; 184 | void add(int p){ n[ncount++]=p; 185 | AddPrediction(squash(p)); 186 | } 187 | }; 188 | Inputs mxInputs[2]; // array of inputs, for two layers 189 | 190 | void Init(){ 191 | y=0 ,c0=1, c4=0,bpos=0,blpos=0,bposshift=0,c0shift_bpos=0 ; 192 | } 193 | }; 194 | 195 | BlockData x; //maintains current global data block 196 | 197 | // ilog(x) = round(log2(x) * 16), 0 <= x < 256 198 | U8 ilog[256]; 199 | // Compute lookup table by numerical integration of 1/x 200 | void InitIlog() { 201 | U32 x=14155776; 202 | for (int i=2; i<257; ++i) { 203 | x+=774541002/(i*2-1); // numerator is 2^29/ln 2 204 | ilog[i-1]=x>>24; 205 | } 206 | } 207 | 208 | // State table 209 | // nex(state, 0) = next state if bit y is 0, 0 <= state < 256 210 | // nex(state, 1) = next state if bit y is 1 211 | // nex(state, 2) = number of zeros in bit history represented by state 212 | // nex(state, 3) = number of ones represented 213 | // 214 | // States represent a bit history within some context. 215 | 216 | // Generating with state parameters p1 28,p2 28,p3 31,p4 29,p5 23,p6 4,p7 17 217 | //Statetable: 218 | static const U8 STA1[256][4]={ 219 | { 1, 2, 0, 0},{ 3, 5, 1, 0},{ 4, 6, 0, 1},{ 7, 9, 2, 0}, // 0-3 220 | { 8, 11, 1, 1},{ 8, 11, 1, 1},{ 10, 12, 0, 2},{ 13, 14, 3, 0}, // 4-7 221 | { 14, 15, 2, 1},{ 14, 15, 2, 1},{ 15, 16, 1, 2},{ 15, 16, 1, 2}, // 8-11 222 | { 16, 17, 0, 3},{ 18, 19, 4, 0},{ 19, 20, 3, 1},{ 20, 21, 2, 2}, // 12-15 223 | { 21, 22, 1, 3},{ 22, 23, 0, 4},{ 24, 25, 5, 0},{ 25, 26, 4, 1}, // 16-19 224 | { 26, 27, 3, 2},{ 27, 28, 2, 3},{ 28, 29, 1, 4},{ 29, 30, 0, 5}, // 20-23 225 | { 31, 32, 6, 0},{ 32, 33, 5, 1},{ 33, 34, 4, 2},{ 34, 35, 3, 3}, // 24-27 226 | { 35, 36, 2, 4},{ 36, 37, 1, 5},{ 37, 38, 0, 6},{ 39, 40, 7, 0}, // 28-31 227 | { 40, 41, 6, 1},{ 41, 42, 5, 2},{ 42, 43, 4, 3},{ 43, 44, 3, 4}, // 32-35 228 | { 44, 45, 2, 5},{ 45, 46, 1, 6},{ 46, 47, 0, 7},{ 48, 49, 8, 0}, // 36-39 229 | { 49, 50, 7, 1},{ 50, 51, 6, 2},{ 51, 52, 5, 3},{ 52, 53, 4, 4}, // 40-43 230 | { 53, 54, 3, 5},{ 54, 55, 2, 6},{ 55, 56, 1, 7},{ 56, 57, 0, 8}, // 44-47 231 | { 58, 59, 9, 0},{ 59, 60, 8, 1},{ 60, 61, 7, 2},{ 61, 62, 6, 3}, // 48-51 232 | { 62, 43, 5, 4},{ 43, 63, 4, 5},{ 63, 64, 3, 6},{ 64, 65, 2, 7}, // 52-55 233 | { 65, 66, 1, 8},{ 66, 67, 0, 9},{ 68, 69,10, 0},{ 69, 70, 9, 1}, // 56-59 234 | { 70, 71, 8, 2},{ 71, 72, 7, 3},{ 72, 52, 6, 4},{ 53, 73, 4, 6}, // 60-63 235 | { 73, 74, 3, 7},{ 74, 75, 2, 8},{ 75, 76, 1, 9},{ 76, 77, 0,10}, // 64-67 236 | { 78, 79,11, 0},{ 79, 80,10, 1},{ 80, 81, 9, 2},{ 81, 82, 8, 3}, // 68-71 237 | { 82, 62, 7, 4},{ 63, 83, 4, 7},{ 83, 84, 3, 8},{ 84, 85, 2, 9}, // 72-75 238 | { 85, 86, 1,10},{ 86, 87, 0,11},{ 88, 89,12, 0},{ 89, 90,11, 1}, // 76-79 239 | { 90, 91,10, 2},{ 91, 92, 9, 3},{ 92, 62, 8, 4},{ 63, 93, 4, 8}, // 80-83 240 | { 93, 94, 3, 9},{ 94, 95, 2,10},{ 95, 96, 1,11},{ 96, 97, 0,12}, // 84-87 241 | { 98, 99,13, 0},{ 99,100,12, 1},{100,101,11, 2},{101,102,10, 3}, // 88-91 242 | {102, 72, 9, 4},{ 73,103, 4, 9},{103,104, 3,10},{104,105, 2,11}, // 92-95 243 | {105,106, 1,12},{106,107, 0,13},{108,109,14, 0},{109,110,13, 1}, // 96-99 244 | {110,111,12, 2},{111,112,11, 3},{112, 82,10, 4},{ 83,113, 4,10}, // 100-103 245 | {113,114, 3,11},{114,115, 2,12},{115,116, 1,13},{116,117, 0,14}, // 104-107 246 | {118,119,15, 0},{119,120,14, 1},{120,121,13, 2},{121,122,12, 3}, // 108-111 247 | {122, 92,11, 4},{ 93,123, 4,11},{123,124, 3,12},{124,125, 2,13}, // 112-115 248 | {125,126, 1,14},{126,127, 0,15},{128,129,16, 0},{129,130,15, 1}, // 116-119 249 | {130,131,14, 2},{131,132,13, 3},{132,102,12, 4},{103,133, 4,12}, // 120-123 250 | {133,134, 3,13},{134,135, 2,14},{135,136, 1,15},{136,137, 0,16}, // 124-127 251 | {138,129,17, 0},{139,140,16, 1},{140,141,15, 2},{141,142,14, 3}, // 128-131 252 | {142,102,13, 4},{103,143, 4,13},{143,144, 3,14},{144,145, 2,15}, // 132-135 253 | {145,146, 1,16},{136,147, 0,17},{148,129,18, 0},{149,140,17, 1}, // 136-139 254 | {150,151,16, 2},{151,152,15, 3},{152,112,14, 4},{113,153, 4,14}, // 140-143 255 | {153,154, 3,15},{154,155, 2,16},{145,156, 1,17},{136,157, 0,18}, // 144-147 256 | {158,129,19, 0},{159,140,18, 1},{160,151,17, 2},{161,162,16, 3}, // 148-151 257 | {162,122,15, 4},{123,163, 4,15},{163,164, 3,16},{154,165, 2,17}, // 152-155 258 | {145,166, 1,18},{136,167, 0,19},{168,129,20, 0},{169,140,19, 1}, // 156-159 259 | {170,151,18, 2},{171,162,17, 3},{172,132,16, 4},{133,173, 4,16}, // 160-163 260 | {163,174, 3,17},{154,175, 2,18},{145,176, 1,19},{136,177, 0,20}, // 164-167 261 | {178,129,21, 0},{179,140,20, 1},{180,151,19, 2},{181,162,18, 3}, // 168-171 262 | {182,132,17, 4},{133,183, 4,17},{163,184, 3,18},{154,185, 2,19}, // 172-175 263 | {145,186, 1,20},{136,187, 0,21},{188,129,22, 0},{189,140,21, 1}, // 176-179 264 | {190,151,20, 2},{191,162,19, 3},{192,132,18, 4},{133,193, 4,18}, // 180-183 265 | {163,194, 3,19},{154,195, 2,20},{145,196, 1,21},{136,197, 0,22}, // 184-187 266 | {198,129,23, 0},{199,140,22, 1},{200,151,21, 2},{201,162,20, 3}, // 188-191 267 | {202,132,19, 4},{133,203, 4,19},{163,204, 3,20},{154,205, 2,21}, // 192-195 268 | {145,206, 1,22},{136,207, 0,23},{208,129,24, 0},{209,140,23, 1}, // 196-199 269 | {210,151,22, 2},{211,162,21, 3},{212,132,20, 4},{133,213, 4,20}, // 200-203 270 | {163,214, 3,21},{154,215, 2,22},{145,216, 1,23},{136,217, 0,24}, // 204-207 271 | {218,129,25, 0},{219,140,24, 1},{220,151,23, 2},{221,162,22, 3}, // 208-211 272 | {222,132,21, 4},{133,223, 4,21},{163,224, 3,22},{154,225, 2,23}, // 212-215 273 | {145,226, 1,24},{136,227, 0,25},{228,129,26, 0},{229,140,25, 1}, // 216-219 274 | {230,151,24, 2},{231,162,23, 3},{161,132,22, 4},{133,164, 4,22}, // 220-223 275 | {163,232, 3,23},{154,233, 2,24},{145,234, 1,25},{136,235, 0,26}, // 224-227 276 | {228,129,27, 0},{236,140,26, 1},{237,151,25, 2},{238,162,24, 3}, // 228-231 277 | {163,239, 3,24},{154,240, 2,25},{145,241, 1,26},{136,235, 0,27}, // 232-235 278 | {236,140,27, 1},{242,151,26, 2},{243,162,25, 3},{163,244, 3,25}, // 236-239 279 | {154,245, 2,26},{145,241, 1,27},{246,151,27, 2},{247,162,26, 3}, // 240-243 280 | {163,248, 3,26},{154,249, 2,27},{250,151,28, 2},{251,162,27, 3}, // 244-247 281 | {163,252, 3,27},{154,253, 2,28},{254,151,29, 2},{170,162,28, 3}, // 248-251 282 | {163,175, 3,28},{154,255, 2,29},{129,151,30, 2},{154,136, 2,30}, // 252-255 283 | }; 284 | 285 | // Generating with state parameters p1 32,p2 28,p3 31,p4 28,p5 21,p6 5,p7 6 286 | //Statetable: 287 | static const U8 STA2[256][4]={ 288 | { 1, 2, 0, 0},{ 3, 5, 1, 0},{ 4, 6, 0, 1},{ 7, 9, 2, 0}, // 0-3 289 | { 8, 11, 1, 1},{ 8, 11, 1, 1},{ 10, 12, 0, 2},{ 13, 15, 3, 0}, // 4-7 290 | { 14, 17, 2, 1},{ 14, 17, 2, 1},{ 16, 19, 1, 2},{ 16, 19, 1, 2}, // 8-11 291 | { 18, 20, 0, 3},{ 21, 22, 4, 0},{ 22, 23, 3, 1},{ 22, 23, 3, 1}, // 12-15 292 | { 23, 24, 2, 2},{ 23, 24, 2, 2},{ 24, 25, 1, 3},{ 24, 25, 1, 3}, // 16-19 293 | { 25, 26, 0, 4},{ 27, 28, 5, 0},{ 28, 29, 4, 1},{ 29, 30, 3, 2}, // 20-23 294 | { 30, 31, 2, 3},{ 31, 32, 1, 4},{ 32, 33, 0, 5},{ 34, 28, 6, 0}, // 24-27 295 | { 35, 36, 5, 1},{ 36, 37, 4, 2},{ 37, 38, 3, 3},{ 38, 39, 2, 4}, // 28-31 296 | { 39, 40, 1, 5},{ 32, 41, 0, 6},{ 42, 28, 7, 0},{ 43, 36, 6, 1}, // 32-35 297 | { 44, 45, 5, 2},{ 45, 46, 4, 3},{ 46, 47, 3, 4},{ 47, 48, 2, 5}, // 36-39 298 | { 39, 49, 1, 6},{ 32, 50, 0, 7},{ 51, 28, 8, 0},{ 52, 36, 7, 1}, // 40-43 299 | { 53, 45, 6, 2},{ 54, 55, 5, 3},{ 55, 56, 4, 4},{ 56, 57, 3, 5}, // 44-47 300 | { 47, 58, 2, 6},{ 39, 59, 1, 7},{ 32, 60, 0, 8},{ 61, 28, 9, 0}, // 48-51 301 | { 62, 36, 8, 1},{ 63, 45, 7, 2},{ 64, 55, 6, 3},{ 65, 46, 5, 4}, // 52-55 302 | { 46, 66, 4, 5},{ 56, 67, 3, 6},{ 47, 68, 2, 7},{ 39, 69, 1, 8}, // 56-59 303 | { 32, 70, 0, 9},{ 71, 28,10, 0},{ 72, 36, 9, 1},{ 73, 45, 8, 2}, // 60-63 304 | { 74, 55, 7, 3},{ 75, 46, 6, 4},{ 46, 76, 4, 6},{ 56, 77, 3, 7}, // 64-67 305 | { 47, 78, 2, 8},{ 39, 79, 1, 9},{ 32, 80, 0,10},{ 81, 28,11, 0}, // 68-71 306 | { 82, 36,10, 1},{ 83, 45, 9, 2},{ 84, 55, 8, 3},{ 85, 46, 7, 4}, // 72-75 307 | { 46, 86, 4, 7},{ 56, 87, 3, 8},{ 47, 88, 2, 9},{ 39, 89, 1,10}, // 76-79 308 | { 32, 90, 0,11},{ 91, 28,12, 0},{ 92, 36,11, 1},{ 93, 45,10, 2}, // 80-83 309 | { 94, 55, 9, 3},{ 95, 46, 8, 4},{ 46, 96, 4, 8},{ 56, 97, 3, 9}, // 84-87 310 | { 47, 98, 2,10},{ 39, 99, 1,11},{ 32,100, 0,12},{101, 28,13, 0}, // 88-91 311 | {102, 36,12, 1},{103, 45,11, 2},{104, 55,10, 3},{105, 46, 9, 4}, // 92-95 312 | { 46,106, 4, 9},{ 56,107, 3,10},{ 47,108, 2,11},{ 39,109, 1,12}, // 96-99 313 | { 32,110, 0,13},{111, 28,14, 0},{112, 36,13, 1},{113, 45,12, 2}, // 100-103 314 | {114, 55,11, 3},{115, 46,10, 4},{ 46,116, 4,10},{ 56,117, 3,11}, // 104-107 315 | { 47,118, 2,12},{ 39,119, 1,13},{ 32,120, 0,14},{121, 28,15, 0}, // 108-111 316 | {122, 36,14, 1},{123, 45,13, 2},{124, 55,12, 3},{125, 46,11, 4}, // 112-115 317 | { 46,126, 4,11},{ 56,127, 3,12},{ 47,128, 2,13},{ 39,129, 1,14}, // 116-119 318 | { 32,130, 0,15},{131, 28,16, 0},{132, 36,15, 1},{133, 45,14, 2}, // 120-123 319 | {134, 55,13, 3},{135, 46,12, 4},{ 46,136, 4,12},{ 56,137, 3,13}, // 124-127 320 | { 47,138, 2,14},{ 39,139, 1,15},{ 32,140, 0,16},{141, 28,17, 0}, // 128-131 321 | {142, 36,16, 1},{143, 45,15, 2},{144, 55,14, 3},{145, 46,13, 4}, // 132-135 322 | { 46,146, 4,13},{ 56,147, 3,14},{ 47,148, 2,15},{ 39,149, 1,16}, // 136-139 323 | { 32,150, 0,17},{151, 28,18, 0},{152, 36,17, 1},{153, 45,16, 2}, // 140-143 324 | {154, 55,15, 3},{155, 46,14, 4},{ 46,156, 4,14},{ 56,157, 3,15}, // 144-147 325 | { 47,158, 2,16},{ 39,159, 1,17},{ 32,160, 0,18},{161, 28,19, 0}, // 148-151 326 | {162, 36,18, 1},{163, 45,17, 2},{164, 55,16, 3},{165, 46,15, 4}, // 152-155 327 | { 46,166, 4,15},{ 56,167, 3,16},{ 47,168, 2,17},{ 39,169, 1,18}, // 156-159 328 | { 32,170, 0,19},{171, 28,20, 0},{172, 36,19, 1},{173, 45,18, 2}, // 160-163 329 | {174, 55,17, 3},{175, 46,16, 4},{ 46,176, 4,16},{ 56,177, 3,17}, // 164-167 330 | { 47,178, 2,18},{ 39,179, 1,19},{ 32,180, 0,20},{181, 28,21, 0}, // 168-171 331 | {182, 36,20, 1},{183, 45,19, 2},{184, 55,18, 3},{185, 46,17, 4}, // 172-175 332 | { 46,186, 4,17},{ 56,187, 3,18},{ 47,188, 2,19},{ 39,189, 1,20}, // 176-179 333 | { 32,190, 0,21},{191, 28,22, 0},{192, 36,21, 1},{193, 45,20, 2}, // 180-183 334 | {194, 55,19, 3},{195, 46,18, 4},{ 46,196, 4,18},{ 56,197, 3,19}, // 184-187 335 | { 47,198, 2,20},{ 39,199, 1,21},{ 32,200, 0,22},{201, 28,23, 0}, // 188-191 336 | {202, 36,22, 1},{203, 45,21, 2},{204, 55,20, 3},{205, 46,19, 4}, // 192-195 337 | { 46,206, 4,19},{ 56,207, 3,20},{ 47,208, 2,21},{ 39,209, 1,22}, // 196-199 338 | { 32,210, 0,23},{211, 28,24, 0},{212, 36,23, 1},{213, 45,22, 2}, // 200-203 339 | {214, 55,21, 3},{154, 46,20, 4},{ 46,157, 4,20},{ 56,215, 3,21}, // 204-207 340 | { 47,216, 2,22},{ 39,217, 1,23},{ 32,218, 0,24},{219, 28,25, 0}, // 208-211 341 | {220, 36,24, 1},{221, 45,23, 2},{222, 55,22, 3},{ 56,223, 3,22}, // 212-215 342 | { 47,224, 2,23},{ 39,225, 1,24},{ 32,226, 0,25},{227, 28,26, 0}, // 216-219 343 | {228, 36,25, 1},{229, 45,24, 2},{230, 55,23, 3},{ 56,231, 3,23}, // 220-223 344 | { 47,232, 2,24},{ 39,233, 1,25},{ 32,234, 0,26},{235, 28,27, 0}, // 224-227 345 | {236, 36,26, 1},{237, 45,25, 2},{238, 55,24, 3},{ 56,239, 3,24}, // 228-231 346 | { 47,240, 2,25},{ 39,241, 1,26},{ 32,242, 0,27},{243, 28,28, 0}, // 232-235 347 | {236, 36,27, 1},{244, 45,26, 2},{245, 55,25, 3},{ 56,246, 3,25}, // 236-239 348 | { 47,247, 2,26},{ 39,241, 1,27},{ 32,248, 0,28},{249, 28,29, 0}, // 240-243 349 | {250, 45,27, 2},{251, 55,26, 3},{ 56,252, 3,26},{ 47,253, 2,27}, // 244-247 350 | { 32,254, 0,29},{255, 28,30, 0},{ 0, 45,28, 2},{173, 55,27, 3}, // 248-251 351 | { 56,178, 3,27},{ 47, 1, 2,28},{ 32, 2, 0,30},{255, 28,31, 0}, // 252-255 352 | }; 353 | 354 | // Generating with state parameters p1 29,p2 30,p3 28,p4 23,p5 29,p6 4,p7 22 355 | //Statetable: 356 | static const U8 STA3[256][4]={ 357 | { 1, 2, 0, 0},{ 3, 5, 1, 0},{ 4, 6, 0, 1},{ 7, 9, 2, 0}, // 0-3 358 | { 8, 11, 1, 1},{ 8, 11, 1, 1},{ 10, 12, 0, 2},{ 13, 14, 3, 0}, // 4-7 359 | { 14, 15, 2, 1},{ 14, 15, 2, 1},{ 15, 16, 1, 2},{ 15, 16, 1, 2}, // 8-11 360 | { 16, 17, 0, 3},{ 18, 19, 4, 0},{ 19, 20, 3, 1},{ 20, 21, 2, 2}, // 12-15 361 | { 21, 22, 1, 3},{ 22, 23, 0, 4},{ 24, 25, 5, 0},{ 25, 26, 4, 1}, // 16-19 362 | { 26, 27, 3, 2},{ 27, 28, 2, 3},{ 28, 29, 1, 4},{ 29, 30, 0, 5}, // 20-23 363 | { 31, 32, 6, 0},{ 32, 33, 5, 1},{ 33, 34, 4, 2},{ 34, 35, 3, 3}, // 24-27 364 | { 35, 36, 2, 4},{ 36, 37, 1, 5},{ 37, 38, 0, 6},{ 39, 40, 7, 0}, // 28-31 365 | { 40, 41, 6, 1},{ 41, 42, 5, 2},{ 42, 43, 4, 3},{ 43, 44, 3, 4}, // 32-35 366 | { 44, 45, 2, 5},{ 45, 46, 1, 6},{ 46, 47, 0, 7},{ 48, 49, 8, 0}, // 36-39 367 | { 49, 50, 7, 1},{ 50, 51, 6, 2},{ 51, 52, 5, 3},{ 52, 53, 4, 4}, // 40-43 368 | { 53, 54, 3, 5},{ 54, 55, 2, 6},{ 55, 56, 1, 7},{ 56, 57, 0, 8}, // 44-47 369 | { 58, 59, 9, 0},{ 59, 60, 8, 1},{ 60, 61, 7, 2},{ 61, 62, 6, 3}, // 48-51 370 | { 62, 43, 5, 4},{ 43, 63, 4, 5},{ 63, 64, 3, 6},{ 64, 65, 2, 7}, // 52-55 371 | { 65, 66, 1, 8},{ 66, 67, 0, 9},{ 68, 69,10, 0},{ 69, 70, 9, 1}, // 56-59 372 | { 70, 71, 8, 2},{ 71, 72, 7, 3},{ 72, 52, 6, 4},{ 53, 73, 4, 6}, // 60-63 373 | { 73, 74, 3, 7},{ 74, 75, 2, 8},{ 75, 76, 1, 9},{ 76, 77, 0,10}, // 64-67 374 | { 78, 79,11, 0},{ 79, 80,10, 1},{ 80, 81, 9, 2},{ 81, 82, 8, 3}, // 68-71 375 | { 82, 62, 7, 4},{ 63, 83, 4, 7},{ 83, 84, 3, 8},{ 84, 85, 2, 9}, // 72-75 376 | { 85, 86, 1,10},{ 86, 87, 0,11},{ 88, 89,12, 0},{ 89, 90,11, 1}, // 76-79 377 | { 90, 91,10, 2},{ 91, 92, 9, 3},{ 92, 62, 8, 4},{ 63, 93, 4, 8}, // 80-83 378 | { 93, 94, 3, 9},{ 94, 95, 2,10},{ 95, 96, 1,11},{ 96, 97, 0,12}, // 84-87 379 | { 98, 99,13, 0},{ 99,100,12, 1},{100,101,11, 2},{101,102,10, 3}, // 88-91 380 | {102, 72, 9, 4},{ 73,103, 4, 9},{103,104, 3,10},{104,105, 2,11}, // 92-95 381 | {105,106, 1,12},{106,107, 0,13},{108,109,14, 0},{109,110,13, 1}, // 96-99 382 | {110,111,12, 2},{111,112,11, 3},{112, 82,10, 4},{ 83,113, 4,10}, // 100-103 383 | {113,114, 3,11},{114,115, 2,12},{115,116, 1,13},{116,117, 0,14}, // 104-107 384 | {118,119,15, 0},{119,120,14, 1},{120,121,13, 2},{121,122,12, 3}, // 108-111 385 | {122, 92,11, 4},{ 93,123, 4,11},{123,124, 3,12},{124,125, 2,13}, // 112-115 386 | {125,126, 1,14},{126,127, 0,15},{128,129,16, 0},{129,130,15, 1}, // 116-119 387 | {130,131,14, 2},{131,132,13, 3},{132,102,12, 4},{103,133, 4,12}, // 120-123 388 | {133,134, 3,13},{134,135, 2,14},{135,136, 1,15},{136,137, 0,16}, // 124-127 389 | {138,139,17, 0},{139,140,16, 1},{140,141,15, 2},{141,142,14, 3}, // 128-131 390 | {142,102,13, 4},{103,143, 4,13},{143,144, 3,14},{144,145, 2,15}, // 132-135 391 | {145,146, 1,16},{146,147, 0,17},{148,149,18, 0},{149,150,17, 1}, // 136-139 392 | {150,151,16, 2},{151,152,15, 3},{152,112,14, 4},{113,153, 4,14}, // 140-143 393 | {153,154, 3,15},{154,155, 2,16},{155,156, 1,17},{156,157, 0,18}, // 144-147 394 | {158,159,19, 0},{159,160,18, 1},{160,161,17, 2},{161,162,16, 3}, // 148-151 395 | {162,122,15, 4},{123,163, 4,15},{163,164, 3,16},{164,165, 2,17}, // 152-155 396 | {165,166, 1,18},{166,167, 0,19},{168,169,20, 0},{169,170,19, 1}, // 156-159 397 | {170,171,18, 2},{171,172,17, 3},{172,132,16, 4},{133,173, 4,16}, // 160-163 398 | {173,174, 3,17},{174,175, 2,18},{175,176, 1,19},{176,177, 0,20}, // 164-167 399 | {178,179,21, 0},{179,180,20, 1},{180,181,19, 2},{181,182,18, 3}, // 168-171 400 | {182,142,17, 4},{143,183, 4,17},{183,184, 3,18},{184,185, 2,19}, // 172-175 401 | {185,186, 1,20},{186,187, 0,21},{188,179,22, 0},{189,190,21, 1}, // 176-179 402 | {190,191,20, 2},{191,192,19, 3},{192,142,18, 4},{143,193, 4,18}, // 180-183 403 | {193,194, 3,19},{194,195, 2,20},{195,196, 1,21},{186,197, 0,22}, // 184-187 404 | {198,179,23, 0},{199,190,22, 1},{200,201,21, 2},{201,202,20, 3}, // 188-191 405 | {202,152,19, 4},{153,203, 4,19},{203,204, 3,20},{204,205, 2,21}, // 192-195 406 | {195,206, 1,22},{186,207, 0,23},{208,179,24, 0},{209,190,23, 1}, // 196-199 407 | {210,201,22, 2},{211,212,21, 3},{212,162,20, 4},{163,213, 4,20}, // 200-203 408 | {213,214, 3,21},{204,215, 2,22},{195,216, 1,23},{186,217, 0,24}, // 204-207 409 | {218,179,25, 0},{219,190,24, 1},{220,201,23, 2},{130,212,22, 3}, // 208-211 410 | {221,172,21, 4},{173,222, 4,21},{213,135, 3,22},{204,223, 2,23}, // 212-215 411 | {195,224, 1,24},{186,225, 0,25},{226,179,26, 0},{227,190,25, 1}, // 216-219 412 | {228,201,24, 2},{229,172,22, 4},{173,230, 4,22},{204,231, 2,24}, // 220-223 413 | {195,232, 1,25},{186,233, 0,26},{234,179,27, 0},{235,190,26, 1}, // 224-227 414 | {236,201,25, 2},{237,172,23, 4},{173,238, 4,23},{204,239, 2,25}, // 228-231 415 | {195,240, 1,26},{186,241, 0,27},{234,179,28, 0},{242,190,27, 1}, // 232-235 416 | {243,201,26, 2},{244,172,24, 4},{173,245, 4,24},{204,246, 2,26}, // 236-239 417 | {195,247, 1,27},{186,241, 0,28},{248,190,28, 1},{109,201,27, 2}, // 240-243 418 | {249,172,25, 4},{173,250, 4,25},{204,116, 2,27},{195,251, 1,28}, // 244-247 419 | {248,190,29, 1},{252,172,26, 4},{173,253, 4,26},{195,251, 1,29}, // 248-251 420 | {254,172,27, 4},{173,255, 4,27},{211,172,28, 4},{173,214, 4,28}, // 252-255 421 | }; 422 | 423 | // Generating with state parameters p1 31,p2 27,p3 30,p4 27,p5 24,p6 4,p7 27 424 | //Statetable: 425 | static const U8 STA4[256][4]={ 426 | { 1, 2, 0, 0},{ 3, 5, 1, 0},{ 4, 6, 0, 1},{ 7, 9, 2, 0}, // 0-3 427 | { 8, 11, 1, 1},{ 8, 11, 1, 1},{ 10, 12, 0, 2},{ 13, 14, 3, 0}, // 4-7 428 | { 14, 15, 2, 1},{ 14, 15, 2, 1},{ 15, 16, 1, 2},{ 15, 16, 1, 2}, // 8-11 429 | { 16, 17, 0, 3},{ 18, 19, 4, 0},{ 19, 20, 3, 1},{ 20, 21, 2, 2}, // 12-15 430 | { 21, 22, 1, 3},{ 22, 23, 0, 4},{ 24, 25, 5, 0},{ 25, 26, 4, 1}, // 16-19 431 | { 26, 27, 3, 2},{ 27, 28, 2, 3},{ 28, 29, 1, 4},{ 29, 30, 0, 5}, // 20-23 432 | { 31, 32, 6, 0},{ 32, 33, 5, 1},{ 33, 34, 4, 2},{ 34, 35, 3, 3}, // 24-27 433 | { 35, 36, 2, 4},{ 36, 37, 1, 5},{ 37, 38, 0, 6},{ 39, 40, 7, 0}, // 28-31 434 | { 40, 41, 6, 1},{ 41, 42, 5, 2},{ 42, 43, 4, 3},{ 43, 44, 3, 4}, // 32-35 435 | { 44, 45, 2, 5},{ 45, 46, 1, 6},{ 46, 47, 0, 7},{ 48, 49, 8, 0}, // 36-39 436 | { 49, 50, 7, 1},{ 50, 51, 6, 2},{ 51, 52, 5, 3},{ 52, 53, 4, 4}, // 40-43 437 | { 53, 54, 3, 5},{ 54, 55, 2, 6},{ 55, 56, 1, 7},{ 56, 57, 0, 8}, // 44-47 438 | { 58, 59, 9, 0},{ 59, 60, 8, 1},{ 60, 61, 7, 2},{ 61, 62, 6, 3}, // 48-51 439 | { 62, 43, 5, 4},{ 43, 63, 4, 5},{ 63, 64, 3, 6},{ 64, 65, 2, 7}, // 52-55 440 | { 65, 66, 1, 8},{ 66, 67, 0, 9},{ 68, 69,10, 0},{ 69, 70, 9, 1}, // 56-59 441 | { 70, 71, 8, 2},{ 71, 72, 7, 3},{ 72, 52, 6, 4},{ 53, 73, 4, 6}, // 60-63 442 | { 73, 74, 3, 7},{ 74, 75, 2, 8},{ 75, 76, 1, 9},{ 76, 77, 0,10}, // 64-67 443 | { 78, 79,11, 0},{ 79, 80,10, 1},{ 80, 81, 9, 2},{ 81, 82, 8, 3}, // 68-71 444 | { 82, 62, 7, 4},{ 63, 83, 4, 7},{ 83, 84, 3, 8},{ 84, 85, 2, 9}, // 72-75 445 | { 85, 86, 1,10},{ 86, 87, 0,11},{ 88, 89,12, 0},{ 89, 90,11, 1}, // 76-79 446 | { 90, 91,10, 2},{ 91, 92, 9, 3},{ 92, 62, 8, 4},{ 63, 93, 4, 8}, // 80-83 447 | { 93, 94, 3, 9},{ 94, 95, 2,10},{ 95, 96, 1,11},{ 96, 97, 0,12}, // 84-87 448 | { 98, 99,13, 0},{ 99,100,12, 1},{100,101,11, 2},{101,102,10, 3}, // 88-91 449 | {102, 72, 9, 4},{ 73,103, 4, 9},{103,104, 3,10},{104,105, 2,11}, // 92-95 450 | {105,106, 1,12},{106,107, 0,13},{108,109,14, 0},{109,110,13, 1}, // 96-99 451 | {110,111,12, 2},{111,112,11, 3},{112, 82,10, 4},{ 83,113, 4,10}, // 100-103 452 | {113,114, 3,11},{114,115, 2,12},{115,116, 1,13},{116,117, 0,14}, // 104-107 453 | {118,119,15, 0},{119,120,14, 1},{120,121,13, 2},{121,122,12, 3}, // 108-111 454 | {122, 92,11, 4},{ 93,123, 4,11},{123,124, 3,12},{124,125, 2,13}, // 112-115 455 | {125,126, 1,14},{126,127, 0,15},{128,129,16, 0},{129,130,15, 1}, // 116-119 456 | {130,131,14, 2},{131,132,13, 3},{132,102,12, 4},{103,133, 4,12}, // 120-123 457 | {133,134, 3,13},{134,135, 2,14},{135,136, 1,15},{136,137, 0,16}, // 124-127 458 | {138,139,17, 0},{139,140,16, 1},{140,141,15, 2},{141,142,14, 3}, // 128-131 459 | {142,102,13, 4},{103,143, 4,13},{143,144, 3,14},{144,145, 2,15}, // 132-135 460 | {145,146, 1,16},{146,147, 0,17},{148,149,18, 0},{149,150,17, 1}, // 136-139 461 | {150,151,16, 2},{151,152,15, 3},{152,112,14, 4},{113,153, 4,14}, // 140-143 462 | {153,154, 3,15},{154,155, 2,16},{155,156, 1,17},{156,157, 0,18}, // 144-147 463 | {158,159,19, 0},{159,160,18, 1},{160,161,17, 2},{161,162,16, 3}, // 148-151 464 | {162,122,15, 4},{123,163, 4,15},{163,164, 3,16},{164,165, 2,17}, // 152-155 465 | {165,166, 1,18},{166,167, 0,19},{168,169,20, 0},{169,170,19, 1}, // 156-159 466 | {170,171,18, 2},{171,172,17, 3},{172,132,16, 4},{133,173, 4,16}, // 160-163 467 | {173,174, 3,17},{174,175, 2,18},{175,176, 1,19},{176,177, 0,20}, // 164-167 468 | {178,179,21, 0},{179,180,20, 1},{180,181,19, 2},{181,182,18, 3}, // 168-171 469 | {182,142,17, 4},{143,183, 4,17},{183,184, 3,18},{184,185, 2,19}, // 172-175 470 | {185,186, 1,20},{186,187, 0,21},{188,189,22, 0},{189,190,21, 1}, // 176-179 471 | {190,191,20, 2},{191,192,19, 3},{192,142,18, 4},{143,193, 4,18}, // 180-183 472 | {193,194, 3,19},{194,195, 2,20},{195,196, 1,21},{196,197, 0,22}, // 184-187 473 | {198,199,23, 0},{199,200,22, 1},{200,201,21, 2},{201,202,20, 3}, // 188-191 474 | {202,152,19, 4},{153,203, 4,19},{203,204, 3,20},{204,205, 2,21}, // 192-195 475 | {205,206, 1,22},{206,207, 0,23},{208,209,24, 0},{209,210,23, 1}, // 196-199 476 | {210,211,22, 2},{211,212,21, 3},{212,162,20, 4},{163,213, 4,20}, // 200-203 477 | {213,214, 3,21},{214,215, 2,22},{215,216, 1,23},{216,217, 0,24}, // 204-207 478 | {218,219,25, 0},{219,220,24, 1},{220,221,23, 2},{221,222,22, 3}, // 208-211 479 | {222,172,21, 4},{173,223, 4,21},{223,224, 3,22},{224,225, 2,23}, // 212-215 480 | {225,226, 1,24},{226,227, 0,25},{228,229,26, 0},{229,230,25, 1}, // 216-219 481 | {230,231,24, 2},{231,232,23, 3},{232,182,22, 4},{183,233, 4,22}, // 220-223 482 | {233,234, 3,23},{234,235, 2,24},{235,236, 1,25},{236,237, 0,26}, // 224-227 483 | {238,229,27, 0},{229,239,26, 1},{239,240,25, 2},{240,171,24, 3}, // 228-231 484 | {171,182,23, 4},{183,174, 4,23},{174,241, 3,24},{241,242, 2,25}, // 232-235 485 | {242,236, 1,26},{236,243, 0,27},{244,229,28, 0},{245,246,26, 2}, // 236-239 486 | {246,181,25, 3},{184,247, 3,25},{247,248, 2,26},{236,249, 0,28}, // 240-243 487 | {250,229,29, 0},{251,246,27, 2},{160,191,26, 3},{194,165, 3,26}, // 244-247 488 | {247,252, 2,27},{236,253, 0,29},{250,229,30, 0},{254,246,28, 2}, // 248-251 489 | {247,255, 2,28},{236,253, 0,30},{119,246,29, 2},{247,126, 2,29}, // 252-255 490 | }; 491 | 492 | // Generating with state parameters p1 33,p2 31,p3 31,p4 24,p5 20,p6 4,p7 33 493 | //Statetable: 494 | static const U8 STA5[256][4]={ 495 | { 1, 2, 0, 0},{ 3, 5, 1, 0},{ 4, 6, 0, 1},{ 7, 9, 2, 0}, // 0-3 496 | { 8, 11, 1, 1},{ 8, 11, 1, 1},{ 10, 12, 0, 2},{ 13, 14, 3, 0}, // 4-7 497 | { 14, 15, 2, 1},{ 14, 15, 2, 1},{ 15, 16, 1, 2},{ 15, 16, 1, 2}, // 8-11 498 | { 16, 17, 0, 3},{ 18, 19, 4, 0},{ 19, 20, 3, 1},{ 20, 21, 2, 2}, // 12-15 499 | { 21, 22, 1, 3},{ 22, 23, 0, 4},{ 24, 25, 5, 0},{ 25, 26, 4, 1}, // 16-19 500 | { 26, 27, 3, 2},{ 27, 28, 2, 3},{ 28, 29, 1, 4},{ 29, 30, 0, 5}, // 20-23 501 | { 31, 32, 6, 0},{ 32, 33, 5, 1},{ 33, 34, 4, 2},{ 34, 35, 3, 3}, // 24-27 502 | { 35, 36, 2, 4},{ 36, 37, 1, 5},{ 37, 38, 0, 6},{ 39, 40, 7, 0}, // 28-31 503 | { 40, 41, 6, 1},{ 41, 42, 5, 2},{ 42, 43, 4, 3},{ 43, 44, 3, 4}, // 32-35 504 | { 44, 45, 2, 5},{ 45, 46, 1, 6},{ 46, 47, 0, 7},{ 48, 49, 8, 0}, // 36-39 505 | { 49, 50, 7, 1},{ 50, 51, 6, 2},{ 51, 52, 5, 3},{ 52, 53, 4, 4}, // 40-43 506 | { 53, 54, 3, 5},{ 54, 55, 2, 6},{ 55, 56, 1, 7},{ 56, 57, 0, 8}, // 44-47 507 | { 58, 59, 9, 0},{ 59, 60, 8, 1},{ 60, 61, 7, 2},{ 61, 62, 6, 3}, // 48-51 508 | { 62, 43, 5, 4},{ 43, 63, 4, 5},{ 63, 64, 3, 6},{ 64, 65, 2, 7}, // 52-55 509 | { 65, 66, 1, 8},{ 66, 67, 0, 9},{ 68, 69,10, 0},{ 69, 70, 9, 1}, // 56-59 510 | { 70, 71, 8, 2},{ 71, 72, 7, 3},{ 72, 52, 6, 4},{ 53, 73, 4, 6}, // 60-63 511 | { 73, 74, 3, 7},{ 74, 75, 2, 8},{ 75, 76, 1, 9},{ 76, 77, 0,10}, // 64-67 512 | { 78, 79,11, 0},{ 79, 80,10, 1},{ 80, 81, 9, 2},{ 81, 82, 8, 3}, // 68-71 513 | { 82, 62, 7, 4},{ 63, 83, 4, 7},{ 83, 84, 3, 8},{ 84, 85, 2, 9}, // 72-75 514 | { 85, 86, 1,10},{ 86, 87, 0,11},{ 88, 89,12, 0},{ 89, 90,11, 1}, // 76-79 515 | { 90, 91,10, 2},{ 91, 92, 9, 3},{ 92, 62, 8, 4},{ 63, 93, 4, 8}, // 80-83 516 | { 93, 94, 3, 9},{ 94, 95, 2,10},{ 95, 96, 1,11},{ 96, 97, 0,12}, // 84-87 517 | { 98, 99,13, 0},{ 99,100,12, 1},{100,101,11, 2},{101,102,10, 3}, // 88-91 518 | {102, 72, 9, 4},{ 73,103, 4, 9},{103,104, 3,10},{104,105, 2,11}, // 92-95 519 | {105,106, 1,12},{106,107, 0,13},{108,109,14, 0},{109,110,13, 1}, // 96-99 520 | {110,111,12, 2},{111,112,11, 3},{112, 82,10, 4},{ 83,113, 4,10}, // 100-103 521 | {113,114, 3,11},{114,115, 2,12},{115,116, 1,13},{116,117, 0,14}, // 104-107 522 | {118,119,15, 0},{119,120,14, 1},{120,121,13, 2},{121,122,12, 3}, // 108-111 523 | {122, 92,11, 4},{ 93,123, 4,11},{123,124, 3,12},{124,125, 2,13}, // 112-115 524 | {125,126, 1,14},{126,127, 0,15},{128,129,16, 0},{129,130,15, 1}, // 116-119 525 | {130,131,14, 2},{131,132,13, 3},{132,102,12, 4},{103,133, 4,12}, // 120-123 526 | {133,134, 3,13},{134,135, 2,14},{135,136, 1,15},{136,137, 0,16}, // 124-127 527 | {138,139,17, 0},{139,140,16, 1},{140,141,15, 2},{141,142,14, 3}, // 128-131 528 | {142,102,13, 4},{103,143, 4,13},{143,144, 3,14},{144,145, 2,15}, // 132-135 529 | {145,146, 1,16},{146,147, 0,17},{148,149,18, 0},{149,150,17, 1}, // 136-139 530 | {150,151,16, 2},{151,152,15, 3},{152,112,14, 4},{113,153, 4,14}, // 140-143 531 | {153,154, 3,15},{154,155, 2,16},{155,156, 1,17},{156,157, 0,18}, // 144-147 532 | {158,159,19, 0},{159,160,18, 1},{160,161,17, 2},{161,162,16, 3}, // 148-151 533 | {162,122,15, 4},{123,163, 4,15},{163,164, 3,16},{164,165, 2,17}, // 152-155 534 | {165,166, 1,18},{166,167, 0,19},{168,169,20, 0},{169,170,19, 1}, // 156-159 535 | {170,171,18, 2},{171,172,17, 3},{172,132,16, 4},{133,173, 4,16}, // 160-163 536 | {173,174, 3,17},{174,175, 2,18},{175,176, 1,19},{176,177, 0,20}, // 164-167 537 | {178,179,21, 0},{179,180,20, 1},{180,181,19, 2},{181,182,18, 3}, // 168-171 538 | {182,142,17, 4},{143,183, 4,17},{183,184, 3,18},{184,185, 2,19}, // 172-175 539 | {185,186, 1,20},{186,187, 0,21},{188,189,22, 0},{189,190,21, 1}, // 176-179 540 | {190,191,20, 2},{191,192,19, 3},{192,142,18, 4},{143,193, 4,18}, // 180-183 541 | {193,194, 3,19},{194,195, 2,20},{195,196, 1,21},{196,197, 0,22}, // 184-187 542 | {198,199,23, 0},{199,200,22, 1},{200,201,21, 2},{201,141,20, 3}, // 188-191 543 | {141,152,19, 4},{153,144, 4,19},{144,202, 3,20},{202,203, 2,21}, // 192-195 544 | {203,204, 1,22},{204,205, 0,23},{206,207,24, 0},{207,208,23, 1}, // 196-199 545 | {208,209,22, 2},{209,151,21, 3},{154,210, 3,21},{210,211, 2,22}, // 200-203 546 | {211,212, 1,23},{212,213, 0,24},{214,215,25, 0},{215,216,24, 1}, // 204-207 547 | {216,217,23, 2},{217,161,22, 3},{164,218, 3,22},{218,219, 2,23}, // 208-211 548 | {219,220, 1,24},{220,221, 0,25},{222,223,26, 0},{223,224,25, 1}, // 212-215 549 | {224,140,24, 2},{140,161,23, 3},{164,145, 3,23},{145,225, 2,24}, // 216-219 550 | {225,226, 1,25},{226,227, 0,26},{228,229,27, 0},{229,230,26, 1}, // 220-223 551 | {230,150,25, 2},{155,231, 2,25},{231,232, 1,26},{232,233, 0,27}, // 224-227 552 | {234,235,28, 0},{235,236,27, 1},{236,150,26, 2},{155,237, 2,26}, // 228-231 553 | {237,238, 1,27},{238,239, 0,28},{240,241,29, 0},{241,242,28, 1}, // 232-235 554 | {242,160,27, 2},{165,243, 2,27},{243,244, 1,28},{244,245, 0,29}, // 236-239 555 | {246,247,30, 0},{247,248,29, 1},{248,170,28, 2},{175,249, 2,28}, // 240-243 556 | {249,250, 1,29},{250,251, 0,30},{252,247,31, 0},{247,253,30, 1}, // 244-247 557 | {253,170,29, 2},{175,254, 2,29},{254,250, 1,30},{250,255, 0,31}, // 248-251 558 | {252,247,32, 0},{129,180,30, 2},{185,136, 2,30},{250,255, 0,32} // 252-255 559 | }; 560 | 561 | 562 | // Mixer m(N, M, S=1, w=0) combines models using M neural networks with 563 | // N inputs each, of which up to S may be selected. If S > 1 then 564 | // the outputs of these neural networks are combined using another 565 | // neural network (with parameters S, 1, 1). If S = 1 then the 566 | // output is direct. The weights are initially w (+-32K). 567 | // It is used as follows: 568 | // m.update() trains the network where the expected output is the 569 | // last bit (in the global variable y). 570 | // m.add(stretch(p)) inputs prediction from one of N models. The 571 | // prediction should be positive to predict a 1 bit, negative for 0, 572 | // nominally +-256 to +-2K. The maximum allowed value is +-32K but 573 | // using such large values may cause overflow if N is large. 574 | // m.set(cxt, range) selects cxt as one of 'range' neural networks to 575 | // use. 0 <= cxt < range. Should be called up to S times such 576 | // that the total of the ranges is <= M. 577 | // m.p() returns the output prediction that the next bit is 1 as a 578 | // 12 bit number (0 to 4095). 579 | 580 | #if !defined(__GNUC__) 581 | 582 | #if (2 == _M_IX86_FP) // 2 if /arch:SSE2 was used. 583 | # define __SSE2__ 584 | #elif (1 == _M_IX86_FP) // 1 if /arch:SSE was used. 585 | # define __SSE__ 586 | #endif 587 | 588 | #endif /* __GNUC__ */ 589 | 590 | #if defined(__AVX2__) 591 | #include 592 | #define OPTIMIZE "AVX2-" 593 | #elif defined(__SSE4_1__) 594 | #include 595 | #elif defined(__SSSE3__) 596 | #include 597 | #elif defined(__SSE2__) 598 | #include 599 | #define OPTIMIZE "SSE2-" 600 | 601 | #elif defined(__SSE__) 602 | #include 603 | #define OPTIMIZE "SSE-" 604 | #endif 605 | 606 | // Vector product a*b of n signed words, returning signed integer scaled down by 8 bits. 607 | // n is rounded up to a multiple of 8. 608 | 609 | //static int dot_product (const short* const t, const short* const w, int n); 610 | 611 | // Train n neural network weights w[n] on inputs t[n] and err. 612 | // w[i] += ((t[i]*2*err)+(1<<16))>>17 bounded to +- 32K. 613 | // n is rounded up to a multiple of 8. 614 | 615 | //static void train (const short* const t, short* const w, int n, const int e); 616 | 617 | #if defined(__MMX__) 618 | typedef __m128i XMM; 619 | #endif 620 | 621 | struct Mixer1 { 622 | int N, M; // max inputs, max contexts, max context sets 623 | short*tx; // N inputs from add() 624 | short* wx ; // N*M weights 625 | short *ptr; 626 | int cxt; // S contexts 627 | int pr; // last result (scaled 12 bits) 628 | int shift1; 629 | int elim; 630 | int uperr; 631 | int err; 632 | #if defined(__AVX2__) 633 | int dot_product (const short* const t, const short* const w, int n) { 634 | assert(n == ((n + 15) & -16)); 635 | __m256i sum = _mm256_setzero_si256 (); 636 | while ((n -= 16) >= 0) { // Each loop sums 16 products 637 | __m256i tmp = _mm256_madd_epi16 (*(__m256i *) &t[n], *(__m256i *) &w[n]); // t[n] * w[n] + t[n+1] * w[n+1] 638 | tmp = _mm256_srai_epi32 (tmp, 8); // (t[n] * w[n] + t[n+1] * w[n+1]) >> 8 639 | sum = _mm256_add_epi32 (sum, tmp); // sum += (t[n] * w[n] + t[n+1] * w[n+1]) >> 8 640 | } 641 | sum =_mm256_hadd_epi32(sum,_mm256_setzero_si256 ()); //add [1]=[1]+[2], [2]=[3]+[4], [3]=0, [4]=0, [5]=[5]+[6], [6]=[7]+[8], [7]=0, [8]=0 642 | sum =_mm256_hadd_epi32(sum,_mm256_setzero_si256 ()); //add [1]=[1]+[2], [2]=0, [3]=0, [4]=0, [5]=[5]+[6], [6]=0, [7]=0, [8]=0 643 | __m128i lo = _mm256_extractf128_si256(sum, 0); 644 | __m128i hi = _mm256_extractf128_si256(sum, 1); 645 | __m128i newsum = _mm_add_epi32(lo, hi); //sum last two 646 | return _mm_cvtsi128_si32(newsum); 647 | } 648 | 649 | void train (const short* const t, short* const w, int n, const int e) { 650 | assert(n == ((n + 15) & -16)); 651 | if (e) { 652 | const __m256i one = _mm256_set1_epi16 (1); 653 | const __m256i err = _mm256_set1_epi16 (short(e)); 654 | while ((n -= 16) >= 0) { // Each iteration adjusts 16 weights 655 | __m256i tmp = _mm256_adds_epi16 (*(__m256i *) &t[n], *(__m256i *) &t[n]); // t[n] * 2 656 | tmp = _mm256_mulhi_epi16 (tmp, err); // (t[n] * 2 * err) >> 16 657 | tmp = _mm256_adds_epi16 (tmp, one); // ((t[n] * 2 * err) >> 16) + 1 658 | tmp = _mm256_srai_epi16 (tmp, 1); // (((t[n] * 2 * err) >> 16) + 1) >> 1 659 | tmp = _mm256_adds_epi16 (tmp, *(__m256i *) &w[n]); // ((((t[n] * 2 * err) >> 16) + 1) >> 1) + w[n] 660 | *(__m256i *) &w[n] = tmp; // save the new eight weights, bounded to +- 32K 661 | } 662 | } 663 | } 664 | 665 | #elif defined(__SSE2__) || defined(__SSSE3__) 666 | int dot_product (const short* const t, const short* const w, int n) { 667 | assert(n == ((n + 15) & -16)); 668 | XMM sum = _mm_setzero_si128 (); 669 | while ((n -= 8) >= 0) { // Each loop sums eight products 670 | XMM tmp = _mm_madd_epi16 (*(XMM *) &t[n], *(XMM *) &w[n]); // t[n] * w[n] + t[n+1] * w[n+1] 671 | tmp = _mm_srai_epi32 (tmp, 8); // (t[n] * w[n] + t[n+1] * w[n+1]) >> 8 672 | sum = _mm_add_epi32 (sum, tmp); // sum += (t[n] * w[n] + t[n+1] * w[n+1]) >> 8 673 | } 674 | #if defined(__SSSE3__) 675 | sum=_mm_hadd_epi32 (sum,sum); 676 | sum=_mm_hadd_epi32 (sum,sum); 677 | #else 678 | sum = _mm_add_epi32(sum, _mm_srli_si128 (sum, 8)); 679 | sum = _mm_add_epi32(sum, _mm_srli_si128 (sum, 4)); 680 | #endif 681 | 682 | return _mm_cvtsi128_si32 (sum); // ... and scale back to integer 683 | } 684 | 685 | void train (const short* const t, short* const w, int n, const int e) { 686 | assert(n == ((n + 15) & -16)); 687 | if (e) { 688 | const XMM one = _mm_set1_epi16 (1); 689 | const XMM err = _mm_set1_epi16 (short(e)); 690 | while ((n -= 8) >= 0) { // Each iteration adjusts eight weights 691 | XMM tmp = _mm_adds_epi16 (*(XMM *) &t[n], *(XMM *) &t[n]); // t[n] * 2 692 | tmp = _mm_mulhi_epi16 (tmp, err); // (t[n] * 2 * err) >> 16 693 | tmp = _mm_adds_epi16 (tmp, one); // ((t[n] * 2 * err) >> 16) + 1 694 | tmp = _mm_srai_epi16 (tmp, 1); // (((t[n] * 2 * err) >> 16) + 1) >> 1 695 | tmp = _mm_adds_epi16 (tmp, *(XMM *) &w[n]); // ((((t[n] * 2 * err) >> 16) + 1) >> 1) + w[n] 696 | *(XMM *) &w[n] = tmp; // save the new eight weights, bounded to +- 32K 697 | } 698 | } 699 | } 700 | 701 | #elif defined(__SSE__) 702 | int dot_product (const short* const t, const short* const w, int n) { 703 | assert(n == ((n + 15) & -16)); 704 | __m64 sum = _mm_setzero_si64 (); 705 | while ((n -= 8) >= 0) { // Each loop sums eight products 706 | __m64 tmp = _mm_madd_pi16 (*(__m64 *) &t[n], *(__m64 *) &w[n]); // t[n] * w[n] + t[n+1] * w[n+1] 707 | tmp = _mm_srai_pi32 (tmp, 8); // (t[n] * w[n] + t[n+1] * w[n+1]) >> 8 708 | sum = _mm_add_pi32 (sum, tmp); // sum += (t[n] * w[n] + t[n+1] * w[n+1]) >> 8 709 | 710 | tmp = _mm_madd_pi16 (*(__m64 *) &t[n + 4], *(__m64 *) &w[n + 4]); // t[n+4] * w[n+4] + t[n+5] * w[n+5] 711 | tmp = _mm_srai_pi32 (tmp, 8); // (t[n+4] * w[n+4] + t[n+5] * w[n+5]) >> 8 712 | sum = _mm_add_pi32 (sum, tmp); // sum += (t[n+4] * w[n+4] + t[n+5] * w[n+5]) >> 8 713 | } 714 | sum = _mm_add_pi32 (sum, _mm_srli_si64 (sum, 32)); // Add eight sums together ... 715 | const int retval = _mm_cvtsi64_si32 (sum); // ... and scale back to integer 716 | _mm_empty(); // Empty the multimedia state 717 | return retval; 718 | } 719 | 720 | void train (const short* const t, short* const w, int n, const int e) { 721 | assert(n == ((n + 15) & -16)); 722 | if (e) { 723 | const __m64 one = _mm_set1_pi16 (1); 724 | const __m64 err = _mm_set1_pi16 (short(e)); 725 | while ((n -= 8) >= 0) { // Each iteration adjusts eight weights 726 | __m64 tmp = _mm_adds_pi16 (*(__m64 *) &t[n], *(__m64 *) &t[n]); // t[n] * 2 727 | tmp = _mm_mulhi_pi16 (tmp, err); // (t[n] * 2 * err) >> 16 728 | tmp = _mm_adds_pi16 (tmp, one); // ((t[n] * 2 * err) >> 16) + 1 729 | tmp = _mm_srai_pi16 (tmp, 1); // (((t[n] * 2 * err) >> 16) + 1) >> 1 730 | tmp = _mm_adds_pi16 (tmp, *(__m64 *) &w[n]); // ((((t[n] * 2 * err) >> 16) + 1) >> 1) + w[n] 731 | *(__m64 *) &w[n] = tmp; // save the new four weights, bounded to +- 32K 732 | 733 | tmp = _mm_adds_pi16 (*(__m64 *) &t[n + 4], *(__m64 *) &t[n + 4]); // t[n+4] * 2 734 | tmp = _mm_mulhi_pi16 (tmp, err); // (t[n+4] * 2 * err) >> 16 735 | tmp = _mm_adds_pi16 (tmp, one); // ((t[n+4] * 2 * err) >> 16) + 1 736 | tmp = _mm_srai_pi16 (tmp, 1); // (((t[n+4] * 2 * err) >> 16) + 1) >> 1 737 | tmp = _mm_adds_pi16 (tmp, *(__m64 *) &w[n + 4]); // ((((t[n+4] * 2 * err) >> 16) + 1) >> 1) + w[n] 738 | *(__m64 *) &w[n + 4] = tmp; // save the new four weights, bounded to +- 32K 739 | } 740 | _mm_empty(); // Empty the multimedia state 741 | } 742 | } 743 | #else 744 | 745 | // dot_product returns dot product t*w of n elements. n is rounded 746 | // up to a multiple of 8. Result is scaled down by 8 bits. 747 | int dot_product(short *t, short *w, int n) { 748 | int sum=0; 749 | n=(n+15)&-16; 750 | for (int i=0; i> 8; 752 | return sum; 753 | } 754 | 755 | // Train neural network weights w[n] given inputs t[n] and err. 756 | // w[i] += t[i]*err, i=0..n-1. t, w, err are signed 16 bits (+- 32K). 757 | // err is scaled 16 bits (representing +- 1/2). w[i] is clamped to +- 32K 758 | // and rounded. n is rounded up to a multiple of 8. 759 | 760 | void train(short *t, short *w, int n, int err) { 761 | n=(n+15)&-16; 762 | for (int i=0; i>16)+1)>>1); 764 | if (wt<-32768) wt=-32768; 765 | if (wt>32767) wt=32767; 766 | w[i]=wt; 767 | } 768 | } 769 | #endif 770 | 771 | // Adjust weights to minimize coding cost of last prediction 772 | void update(int y) { 773 | err=((y<<12)-pr)*uperr/4; 774 | if (err>32767) 775 | err=32767; 776 | if (err<-32768) 777 | err=-32768; 778 | if(err>=-elim && err<=elim) err=0; 779 | train(&tx[0], &wx[cxt*N], N, err); 780 | } 781 | 782 | // predict next bit 783 | int p( ) { 784 | assert(cxt>11; 786 | return pr=squash(dp); 787 | } 788 | int p1( ) { 789 | assert(cxt>11; 791 | if (dp<-2047) { 792 | dp=-2047; 793 | } 794 | else if (dp>2047) { 795 | dp=2047; 796 | } 797 | pr=squash(dp); 798 | return dp; 799 | } 800 | void setTxWx(int n,short* mn){ 801 | N=n; 802 | alloc1(wx,(N*M)+32,ptr,32); 803 | tx=mn; 804 | } 805 | void Init(int m, U32 s,U32 e,U32 ue){ 806 | M=m, cxt=0, shift1=s,elim=e,uperr=ue;err=0; 807 | pr=2048; //initial p=0.5 808 | } 809 | void Free(){ 810 | free(ptr); 811 | } 812 | /* void Print(){ 813 | // print N weights averaged over context 814 | printf("Mixer(%d,%d): ", N, M); 815 | for (int i=0; i 16K/(i+i+3) 834 | 835 | struct StateMap { 836 | int N; // Number of contexts 837 | int cxt; // Context of last prediction 838 | U32 *t; // cxt -> prediction in high 22 bits, count in low 10 bits 839 | int pr; 840 | int mask; 841 | int limit; 842 | const U8 *nn; 843 | int next(int i, int y){ 844 | return nn[ y + i*4]; 845 | } 846 | void Init(int n, int lim,const U8 *nn1){nn=nn1; 847 | N=n, cxt=0, pr=2048, mask=n-1,limit=lim; 848 | assert(ispowerof2(n)); 849 | alloc(t,n); 850 | assert(limit>0 && limit<1024); 851 | if (N==256){ 852 | for (int i=0; i>12; // count, prediction 869 | p0+=(n=0 && cxt>20; 878 | } 879 | /*void print(){ 880 | for (int i=0;i>20); 882 | } 883 | }*/ 884 | }; 885 | 886 | inline short clp(int z){ 887 | if (z<-2047){ 888 | z=-2047; 889 | }else if (z>2047){ 890 | z=2047; 891 | } 892 | return z; 893 | } 894 | inline short clp1(int z){ 895 | if (z<0){ 896 | z=0; 897 | }else if (z>4095){ 898 | z=4095; 899 | } 900 | return z; 901 | } 902 | // A RunContextMap maps a context into the next byte and a repeat 903 | // count up to M. Size should be a power of 2. Memory usage is 3M/4. 904 | struct RunContextMap { 905 | enum {B=4,M=4}; 906 | U8 *t; // hash t 907 | U8 *ptr; 908 | U8* cp; 909 | short rc[512]; 910 | U8 tmp[B]; 911 | U32 n; 912 | void Init(int m,int rcm_ml=8){ 913 | alloc1(t,m,ptr,64); 914 | n=(m/B-1); 915 | for (int r=0;r> x.bposshift); 936 | if (b<=1) 937 | return rc[b*256+cp[0]]; 938 | else 939 | return 0; 940 | } 941 | int mix(int m) { // return run length 942 | x.mxInputs[m].add(p()); 943 | return cp[0]!=0; 944 | } 945 | 946 | inline U8* find(U32 i) { 947 | U16 chk=(i>>16^i)&0xffff; 948 | i=i*M&n; 949 | U8 *p; 950 | U16 *cp1; 951 | int j; 952 | for (j=0; j2 && t[(i+j)*B+2]>t[(i+j-1)*B+2]) --j; 964 | } 965 | else memcpy(&tmp, cp1, B); 966 | memmove(&t[(i+1)*B], &t[i*B], j*B); 967 | memcpy(&t[i*B], &tmp, B); 968 | return &t[i*B+1]; 969 | } 970 | }; 971 | 972 | // Map for modelling contexts of (nearly-)stationary data. 973 | // The context is looked up directly. For each bit modelled, a 16bit prediction is stored. 974 | // The adaptation rate is controlled by the caller, see mix(). 975 | 976 | // - BitsOfContext: How many bits to use for each context. Higher bits are discarded. 977 | // - InputBits: How many bits [1..8] of input are to be modelled for each context. 978 | // New contexts must be set at those intervals. 979 | 980 | // Uses (2^(BitsOfContext+1))*((2^InputBits)-1) bytes of memory. 981 | 982 | struct SmallStationaryContextMap { 983 | U16 *Data; 984 | int Context, Mask, Stride, bCount, bTotal, B,N; 985 | U16 *cp; 986 | 987 | void Init(int BitsOfContext, int InputBits = 8) { 988 | assert(InputBits>0 && InputBits<=8); 989 | Context=0, Mask=((1<>rate; 1007 | B+=(x.y && B>0); 1008 | cp = &Data[Context+B]; 1009 | int Prediction = (*cp)>>4; 1010 | x.mxInputs[m].add((stretch(Prediction)*Multiplier)/Divisor); 1011 | x.mxInputs[m].add(((Prediction-2048)*Multiplier)/(Divisor*2)); 1012 | bCount++; B+=B+1; 1013 | if (bCount==bTotal) 1014 | bCount=B=0; 1015 | } 1016 | }; 1017 | 1018 | 1019 | 1020 | // Context map for large contexts. Most modeling uses this type of context 1021 | // map. It includes a built in RunContextMap to predict the last byte seen 1022 | // in the same context, and also bit-level contexts that map to a bit 1023 | // history state. 1024 | // 1025 | // Bit histories are stored in a hash table. The table is organized into 1026 | // 64-byte buckets alinged on cache page boundaries. Each bucket contains 1027 | // a hash chain of 7 elements, plus a 2 element queue (packed into 1 byte) 1028 | // of the last 2 elements accessed for LRU replacement. Each element has 1029 | // a 2 byte checksum for detecting collisions, and an array of 7 bit history 1030 | // states indexed by the last 0 to 2 bits of context. The buckets are indexed 1031 | // by a context ending after 0, 2, or 5 bits of the current byte. Thus, each 1032 | // byte modeled results in 3 main memory accesses per context, with all other 1033 | // accesses to cache. 1034 | // 1035 | // On bits 0, 2 and 5, the context is updated and a new bucket is selected. 1036 | // The most recently accessed element is tried first, by comparing the 1037 | // 16 bit checksum, then the 7 elements are searched linearly. If no match 1038 | // is found, then the element with the lowest priority among the 5 elements 1039 | // not in the LRU queue is replaced. After a replacement, the queue is 1040 | // emptied (so that consecutive misses favor a LFU replacement policy). 1041 | // In all cases, the found/replaced element is put in the front of the queue. 1042 | // 1043 | // The priority is the state number of the first element (the one with 0 1044 | // additional bits of context). The states are sorted by increasing n0+n1 1045 | // (number of bits seen), implementing a LFU replacement policy. 1046 | // 1047 | // When the context ends on a byte boundary (bit 0), only 3 of the 7 bit 1048 | // history states are used. The remaining 4 bytes implement a run model 1049 | // as follows: where is the last byte 1050 | // seen, possibly repeated. is a 7 bit count and a 1 bit 1051 | // flag (represented by count * 2 + d). If d=0 then = 1..127 is the 1052 | // number of repeats of and no other bytes have been seen. If d is 1 then 1053 | // other byte values have been seen in this context prior to the last 1054 | // copies of . 1055 | // 1056 | // As an optimization, the last two hash elements of each byte (representing 1057 | // contexts with 2-7 bits) are not updated until a context is seen for 1058 | // a second time. This is indicated by = <1,0> (2). After update, 1059 | // is updated to <2,0> or <1,1> (4 or 3). 1060 | 1061 | inline int sc(int p){ 1062 | if (p>0) return p>>7; 1063 | return (p+127)>>7;// p+((1< bit history state 1089 | // bh[][0] = 1st bit, bh[][1,2] = 2nd bit, bh[][3..6] = 3rd bit 1090 | // bh[][0] is also a replacement priority, 0 = empty 1091 | // U8* get(U16 chk); // Find element (0-6) matching checksum. 1092 | // If not found, insert or replace lowest priority (not last). 1093 | inline U8* get(U16 ch,int keep) { 1094 | 1095 | if (chk[last&15]==ch) return &bh[last&15][0]; 1096 | int b=0xffff, bi=0; 1097 | 1098 | for (int i=0; i<7; ++i) { 1099 | if (chk[i]==ch) return last=last<<4|i, (U8*)&bh[i][0]; 1100 | int pri=bh[i][0]; 1101 | if (pri>4!=i) b=pri, bi=i; 1102 | } 1103 | return last=last<<4|bi|keep, chk[bi]=ch, (U8*)memset(&bh[bi][0], 0, 7); 1104 | } 1105 | 1106 | }; 1107 | 1108 | inline U32 getStateByteLocation(const int bpos, const int c0) { 1109 | U32 pis = 0; //state byte position in slot 1110 | const U32 smask = (U32(0x31031010) >> (bpos << 2)) & 0x0F; 1111 | pis = smask + (c0 & smask); 1112 | return pis; 1113 | } 1114 | 1115 | #define MAXCXT 8 1116 | 1117 | struct ContextMap { 1118 | int C; // max number of contexts 1119 | U8* cp[MAXCXT]; // C pointers to current bit history 1120 | U8* cp0[MAXCXT]; // First element of 7 element array containing cp[i] 1121 | U32 cxt[MAXCXT]; // C whole byte contexts (hashes) 1122 | U8* runp[MAXCXT]; // C [0..3] = count, value, unused, unused 1123 | StateMap *sm; // C maps of state -> p 1124 | int cn; // Next context to set by set() 1125 | int result; 1126 | short rc1[512]; 1127 | short st1[4096]; 1128 | short st2[4096]; 1129 | short st32[256]; 1130 | short st8[256]; 1131 | int cms,cms2,cms3,cms4; 1132 | int kep; 1133 | const U8 *nn; 1134 | E *ptr,*t; 1135 | U32 tmask; 1136 | int skip2; 1137 | const inline U8 next(int i, int y){ 1138 | return nn[ y + i*4]; 1139 | } 1140 | 1141 | int __attribute__ ((noinline)) mix(const int m) {return mix1(m, x.c0, x.bpos, (U8) x.c4, x.y);} 1142 | inline int pre(const int state) { 1143 | assert(state>=0 && state<256); 1144 | U32 n0=next(state, 2)*3+1; 1145 | U32 n1=next(state, 3)*3+1; 1146 | return (n1<<12) / (n0+n1); 1147 | } 1148 | 1149 | // Construct using m bytes of memory for c contexts(c+7)&-8 1150 | void __attribute__ ((noinline)) Init(U32 m, int c, int s3,const U8 *nn1,int cs4,int k,int u){ 1151 | C=c&255; 1152 | tmask=((m>>6)-1); 1153 | cn=0; 1154 | result=0; 1155 | kep=k; 1156 | alloc1(t,(m>>6)+64,ptr,64); 1157 | nn=nn1; 1158 | int cmul=(c>>8)&255; // run context mul value 1159 | cms=(c>>16)&255; // mix prediction mul value 1160 | cms2=(U32(c)>>24)&255; 1161 | cms4=cs4; 1162 | cms3=s3; 1163 | skip2=u; 1164 | assert(m>=64 && (m&m-1)==0); // power of 2? 1165 | assert(sizeof(E)==64); 1166 | alloc(sm,C); 1167 | for (int i=0; i=0 && i>16; 1220 | cxt[i]=cx*123456791+i; 1221 | } 1222 | 1223 | // Predict to mixer m from bit history state s, using sm to map s to 1224 | // a probability. 1225 | inline int mix3(const int y,const int m,const int s, StateMap& sm) { 1226 | if (s==0){ 1227 | x.mxInputs[m].add(0); 1228 | if (skip2==1)x.mxInputs[m].add(0); 1229 | x.mxInputs[m].add(0); 1230 | x.mxInputs[m].add(0); 1231 | x.mxInputs[m].add(32*2); 1232 | return 0; 1233 | }else{ 1234 | sm.set(s,y); 1235 | const int p1=sm.pr; 1236 | x.mxInputs[m].add(st1[p1]); // From StateMap 1237 | if (skip2==1)x.mxInputs[m].add(st2[p1]); 1238 | x.mxInputs[m].add(st8[s]); // From state 1239 | x.mxInputs[m].add(st32[s]); 1240 | x.mxInputs[m].add(0); 1241 | return 1; 1242 | } 1243 | } 1244 | 1245 | // Zero prediction 1246 | inline void mix4(int m) { 1247 | x.mxInputs[m].add(0); 1248 | x.mxInputs[m].add(0); 1249 | x.mxInputs[m].add(0); 1250 | x.mxInputs[m].add(0); 1251 | x.mxInputs[m].add(32*2); 1252 | x.mxInputs[m].add(0); 1253 | } 1254 | 1255 | // Update the model with bit y1, and predict next bit to mixer m. 1256 | // Context: cc=c0, bp=bpos, c1=buf(1), y1=y. 1257 | int mix1(const int m,const int cc,const int bp,const int c1,const int y1) { 1258 | // Update model with y 1259 | result=0; 1260 | 1261 | for (int i=0; i=&t[0].bh[0][0] && cp[i]<=&t[tmask].bh[6][6]); 1264 | assert(((long long)(cp[i])&63)>=15); 1265 | *cp[i]=next(*cp[i], y1); 1266 | } 1267 | 1268 | // Update context pointers 1269 | int s = 0; 1270 | if (bp>1 && runp[i][0]==0) { 1271 | cp[i]=0; 1272 | } else { 1273 | U16 chksum=(cxt[i]>>16)^i; 1274 | 1275 | if (bp){ 1276 | if (bp==2 || bp==5)cp0[i]=cp[i]=t[(cxt[i]+cc)&tmask].get(chksum,kep); 1277 | else cp[i]=cp0[i]+getStateByteLocation(bp,cc); 1278 | } else {// default 1279 | cp0[i]=cp[i]=t[(cxt[i]+cc)&tmask].get(chksum,kep); 1280 | // Update pending bit histories for bits 2-7 1281 | if (cp0[i][3]==2) { 1282 | const int c=cp0[i][4]+256; 1283 | U8 *p=t[(cxt[i]+(c>>6))&tmask].get(chksum,kep); 1284 | p[0]=1+((c>>5)&1); 1285 | p[1+((c>>5)&1)]=1+((c>>4)&1); 1286 | p[3+((c>>4)&3)]=1+((c>>3)&1); 1287 | p=t[(cxt[i]+(c>>3))&tmask].get(chksum,kep); 1288 | p[0]=1+((c>>2)&1); 1289 | p[1+((c>>2)&1)]=1+((c>>1)&1); 1290 | p[3+((c>>1)&3)]=1+(c&1); 1291 | cp0[i][6]=0; 1292 | } 1293 | // Update run count of previous context 1294 | if (runp[i][0]==0) // new context 1295 | runp[i][0]=2, runp[i][1]=c1; 1296 | else if (runp[i][1]!=c1) // different byte in context 1297 | runp[i][0]=1, runp[i][1]=c1; 1298 | else if (runp[i][0]<254) // same byte in context 1299 | runp[i][0]+=2; 1300 | runp[i]=cp0[i]+3; 1301 | } 1302 | s = *cp[i]; 1303 | } 1304 | // predict from bit context 1305 | 1306 | result=result+mix3(y1,m, s, sm[i]); 1307 | // predict from last byte in context 1308 | int b=x.c0shift_bpos ^ (runp[i][1] >> x.bposshift); 1309 | if (b<=1) { 1310 | b=b*256; // predicted bit + for 1, - for 0 1311 | // count*2, +1 if 2 different bytes seen 1312 | x.mxInputs[m].add(rc1[runp[i][0]+b]); 1313 | } 1314 | else 1315 | x.mxInputs[m].add(0); 1316 | } 1317 | if (bp==7) cn=0; 1318 | return result; 1319 | } 1320 | }; 1321 | 1322 | // APM maps a probability and a context into a new probability 1323 | // that bit y will next be 1. After each guess it updates 1324 | // its state to improve future guesses. Methods: 1325 | // 1326 | // APM a(N) creates with N contexts, uses 66*N bytes memory. 1327 | // a.p(pr, cx, rate=8) returned adjusted probability in context cx (0 to 1328 | // N-1). rate determines the learning rate (smaller = faster, default 8). 1329 | // Probabilities are scaled 16 bits (0-65535). 1330 | 1331 | struct APM { 1332 | int index; // last p, context 1333 | U16 *t; // [N][33]: p, context -> p 1334 | 1335 | int p(int pr=2048, int cxt=0, int rate=8, int y=0) { 1336 | pr=stretch(pr); 1337 | int g=(y<<16)+(y<> rate; 1339 | t[index+1] += (g-t[index+1]) >> rate; 1340 | const int w=pr&127; // interpolation weight (33 points) 1341 | index=((pr+2048)>>7)+cxt*33; 1342 | return (t[index]*(128-w)+t[index+1]*w) >> 11; 1343 | } 1344 | 1345 | // maps p, cxt -> p initially 1346 | void Init(int n){ 1347 | index=0; 1348 | alloc(t,n*33); 1349 | for (int j=0; j<33; ++j) t[j]=squash((j-16)*128)*16; 1350 | for (int i=33; i? 1363 | 2, 2, 0, 0, 2, 3, 1, 2, 1, 2, 2, 2, 2, 2, 0, 0, // @ABCDEFGHIJKLMNO 1364 | 2, 2, 2, 2, 2, 2, 2, 2, 3, 0, 2, 3, 2, 0, 2, 3, // PQRSTUVWXYZ[\]^_ 1365 | 1366 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // abcdefghijklmno 1367 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // pqrstuvwxyz{|}~ 1368 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1369 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1370 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1371 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1372 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1373 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1374 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1375 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 1376 | 1377 | static const U8 wrt_t[256]={ 1378 | 0, 0, 2, 0, 5, 6, 0, 6, 0, 2, 0, 4, 3, 0, 0, 0, 1379 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1380 | 2, 4, 1, 4, 4, 7, 4, 7, 3, 7, 2, 2, 3, 5, 3, 1, // _!"#$%&'()*+,-./ 1381 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 5, 3, 3, 5, 5, // 0123456789:;<=>? 1382 | 0, 5, 5, 7, 5, 0, 1, 5, 4, 5, 0, 0, 6, 0, 7, 1, // @ABCDEFGHIJKLMNO 1383 | 3, 3, 7, 4, 5, 5, 7, 0, 2, 2, 5, 4, 4, 7, 4, 6, // PQRSTUVWXYZ[\]^_ 1384 | 1385 | 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1386 | 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1387 | 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 1388 | 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 1389 | 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 1390 | 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 1391 | 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 1392 | 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 1393 | 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 1394 | 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7}; 1395 | 1396 | 1397 | #ifdef TEXTMODE 1398 | //text 1399 | #define COLON 58 // : 1400 | #define SEMICOLON 59 // ; 1401 | #define LESSTHAN 60 // < 1402 | #define EQUALS 61 // = 1403 | #define GREATERTHAN 62 // > 1404 | #define QUESTION 63 // ? 1405 | #define ATSIGN 64 // @ 1406 | #define SQUAREOPEN 91 // [ 1407 | #define BACKSLASH 92 // '\' 1408 | #define SQUARECLOSE 93 // ] 1409 | #define CURLYOPENING 123 // { 1410 | #define VERTICALBAR 124 // | 1411 | #define CURLYCLOSE 125 // } 1412 | #else 1413 | //wrt 1414 | #define COLON 'J' // : 1415 | #define SEMICOLON 'K' // ; 1416 | #define LESSTHAN 'L' // < 1417 | #define EQUALS 'M' // = 1418 | #define GREATERTHAN 'N' // > 1419 | #define QUESTION 'O' // ? 1420 | #define ATSIGN 64 // @ 1421 | #define SQUAREOPEN 91 // [ 1422 | #define BACKSLASH 92 // '\' 1423 | #define SQUARECLOSE 93 // ] 1424 | #define CURLYOPENING 'P' // { 1425 | #define VERTICALBAR 'Q' // | 1426 | #define CURLYCLOSE 'R' // } 1427 | #define CHARSWAP 1428 | #endif 1429 | 1430 | #define APOSTROPHE 39 // ' 1431 | #define QUOTATION 34 // " 1432 | #define SPACE 32 // ' ' 1433 | 1434 | // Vector for different contexts 1435 | static const int charSize = 32; 1436 | template 1437 | struct vec { 1438 | T* cxt; 1439 | int capacity; 1440 | int size; 1441 | }; 1442 | 1443 | template 1444 | void vec_new(vec* o){ 1445 | o->cxt=static_cast( calloc(charSize, sizeof(T))); 1446 | o->capacity=charSize; 1447 | o->size=0; 1448 | } 1449 | template 1450 | void vec_free(vec* o){ 1451 | free(o->cxt); 1452 | } 1453 | template 1454 | int vec_size(vec *o){ 1455 | return o->size; 1456 | } 1457 | template 1458 | void vec_push(struct vec *o, const T element){ 1459 | if(o->size>0 && o->size%o->capacity==0) { 1460 | o->capacity=(o->size/charSize+1)*charSize; 1461 | o->cxt=static_cast(realloc(o->cxt, o->capacity*sizeof(T))); 1462 | } 1463 | o->cxt[o->size++]=element; 1464 | } 1465 | template 1466 | int vec_at(vec *o, const int index){ 1467 | return o->cxt[index]; 1468 | } 1469 | template 1470 | void vec_i(vec *o, const int index){ 1471 | o->cxt[index]++; 1472 | } 1473 | template 1474 | void vec_pop(vec *o){ 1475 | o->cxt[o->size-1]=0; 1476 | o->size--; 1477 | } 1478 | template 1479 | void vec_reset(vec *o){ 1480 | o->cxt[0]=0; 1481 | o->size=0; 1482 | } 1483 | template 1484 | bool vec_empty(vec *o){ 1485 | return (o->size==0)?true:false; 1486 | } 1487 | template 1488 | int vec_prev(vec *o){ 1489 | return (o->size>1)?(o->cxt[o->size-2]):0; 1490 | } 1491 | // This part is based on cmix BracketContext 1492 | struct BracketContext { 1493 | U32 context; // bracket byte and distance 1494 | vec active; // vector for brackets 1495 | vec distance; // vector for distance 1496 | const U8 *element; 1497 | int elementCount; 1498 | bool doPop; // set true for quotes 1499 | int limit; 1500 | U8 cxt,dst; 1501 | 1502 | void Init(const U8*d,const int e,int pop=false,int l=255) { 1503 | elementCount=e; 1504 | element=d; 1505 | context=cxt=dst=0; 1506 | doPop=pop; 1507 | limit=l; 1508 | vec_new(&active); 1509 | vec_new(&distance); 1510 | } 1511 | void Reset(){ 1512 | vec_reset(&active); 1513 | vec_reset(&distance); 1514 | context=cxt=dst=0; 1515 | } 1516 | bool Find(int b){ 1517 | bool found=false; 1518 | for (int i=0;i= limit) { 1536 | vec_pop(&active); 1537 | vec_pop(&distance); 1538 | pop=doPop; 1539 | } else { 1540 | vec_i(&distance,vec_size(&distance)-1); 1541 | } 1542 | } 1543 | if (pop==false && Find(byte)) { 1544 | vec_push( &active,byte); 1545 | vec_push( &distance,0); 1546 | } 1547 | if (!vec_empty(&active)) { 1548 | cxt=vec_at(&active,vec_size(&active)-1); 1549 | dst=min(vec_at(&distance,vec_size(&distance)-1),255); 1550 | context = 256 * cxt+dst; 1551 | } else { 1552 | context=cxt=dst=0; 1553 | } 1554 | } 1555 | void Free(){ 1556 | vec_free(&active); 1557 | vec_free(&distance); 1558 | } 1559 | }; 1560 | 1561 | // Table/row & column context 1562 | struct Column { 1563 | U32 linepos; 1564 | U8 fc; 1565 | vec bytes; 1566 | }; 1567 | 1568 | struct ColumnContext { 1569 | Column col[4]; // Content of last 3 + current row 1570 | vec cell[4]; // Content of table row cell positions, max 4 rows 1571 | int rows; 1572 | int cellCount,cells,abovecellpos,abovecellpos1; 1573 | bool NL,isTemp; 1574 | int limit; // column lenght limit 1575 | U8 nlChar; 1576 | void Init( int l=31) { 1577 | rows=abovecellpos=cellCount=abovecellpos1=0; 1578 | nlChar=10; 1579 | limit=l; 1580 | for (int i=0;i<4;i++) vec_new(&col[i].bytes); 1581 | for (int i=0;i<4;i++) vec_new(&cell[i]); 1582 | NL=isTemp=false; 1583 | } 1584 | 1585 | U8 lastfc(int i=0){ 1586 | return col[(rows-i)&3].fc; 1587 | } 1588 | bool isNewLine(){ 1589 | return NL; 1590 | } 1591 | int collen(int i=0,int l=0){ 1592 | return min((l?l:limit), vec_size(&col[(rows-i)&3].bytes)+1); 1593 | } 1594 | int nlpos(int i=0){ 1595 | return col[(rows-i)&3].linepos; 1596 | } 1597 | U8 __attribute__ ((noinline)) colb(int i=1,int j=0,int l=0){ 1598 | if (collen(0,l)abovecellpos1) abovecellpos=abovecellpos1=0; 1655 | } 1656 | // If more then one cell get above cell based on current row cell 1657 | if(newcell==true && cellsCount() >0){ 1658 | // Get current above cell pos 1659 | abovecellpos=cellPos(cellCount-1); 1660 | abovecellpos1=cellPos(cellCount); 1661 | } 1662 | } 1663 | #ifndef TEXTMODE 1664 | if (nlChar==GREATERTHAN){ 1665 | if ((b2&0xffff)==(GREATERTHAN+10*256)){ 1666 | //printf("%d %d\n",vec_size(&cell[cells]),vec_at(&cell[cells],vec_size(&cell[cells])-1)); 1667 | cells++; 1668 | cells=cells&3; 1669 | vec_reset(&cell[cells]); // reset new row. 1670 | vec_push( &cell[cells],U32(x.blpos)); 1671 | cellCount=abovecellpos=abovecellpos1=0; 1672 | //printf("\n"); 1673 | }else{ 1674 | 1675 | bool newcell=false; 1676 | // Cells 1677 | if ( (b2&0xff)==(GREATERTHAN) ) vec_push( &cell[cells],U32(x.blpos)),cellCount++,newcell=true; 1678 | // Advence above cell pos 1679 | if (abovecellpos ) { 1680 | abovecellpos++; 1681 | // When above cell is shorter reset 1682 | if (abovecellpos>abovecellpos1) abovecellpos=abovecellpos1=0; 1683 | } 1684 | // If more then one cell get above cell based on current row cell 1685 | if(newcell==true && cellsCount() >0){ 1686 | //printf("%d ",cellPos(cellCount-1)); 1687 | // Get current above cell pos 1688 | abovecellpos=cellPos(cellCount-1); 1689 | abovecellpos1=cellPos(cellCount); 1690 | } 1691 | } 1692 | } 1693 | #endif 1694 | } 1695 | int cellsCount(int row=1){ 1696 | return vec_size(&cell[(cells-row)&3]); 1697 | } 1698 | int cellPos(int cellID,int row=1){ 1699 | int total=cellsCount(row)-1; 1700 | total=min(total,cellID); 1701 | return vec_at(&cell[(cells-row)&3],total); 1702 | } 1703 | void resetCells(){ 1704 | for (int i=0;i<4;i++) vec_reset(&cell[i]); 1705 | } 1706 | void Free(){ 1707 | for (int i=0;i<4;i++) vec_free(&col[i].bytes); 1708 | for (int i=0;i<4;i++) vec_reset(&cell[i]); 1709 | } 1710 | }; 1711 | // Keep track of main brackets 1712 | const U8 brackets[8]={'(',')', CURLYOPENING,CURLYCLOSE, '[',']', LESSTHAN,GREATERTHAN}; 1713 | // Keep track of ' and " as quotes 1714 | const U8 quotes[4]={APOSTROPHE,APOSTROPHE,QUOTATION,QUOTATION}; 1715 | // Keep track of first char including some brackets 1716 | const U8 fchar[20]={ATSIGN,10, 96,10, COLON,10, LESSTHAN,GREATERTHAN,EQUALS,10,SQUAREOPEN,SQUARECLOSE,CURLYOPENING,CURLYCLOSE,'*',10,VERTICALBAR,10,31,10}; 1717 | 1718 | // Sentence & words context 1719 | struct WordsContext { 1720 | vec awords; // List of words 1721 | vec sbytes; // List of bytes surrounded by a current word 1722 | U32 fword; // First word 1723 | U8 pbyte; // Current byte before word 1724 | void Init() { 1725 | vec_new(&awords); 1726 | vec_new(&sbytes); 1727 | } 1728 | void Reset(){ 1729 | vec_reset(&awords); 1730 | vec_reset(&sbytes); 1731 | fword=pbyte=0; 1732 | } 1733 | void Set(U8 b){ 1734 | pbyte=b; 1735 | } 1736 | void __attribute__ ((noinline)) Update(U32 w,U8 b) { 1737 | if (fword==0) fword=w; 1738 | vec_push(&awords,w); 1739 | vec_push(&sbytes,U16(pbyte*256+b)); // Surrounding bytes 1740 | pbyte=0; 1741 | } 1742 | void __attribute__ ((noinline)) Remove(){ 1743 | int num=vec_size(&awords); 1744 | if (num) vec_pop(&awords),vec_pop(&sbytes); 1745 | } 1746 | U32 __attribute__ ((noinline)) Word(int i=1){ 1747 | int num=vec_size(&awords); 1748 | if (num>=i) return vec_at(&awords,num-(i)); 1749 | else return 0; 1750 | } 1751 | U16 sBytes(int i=1){ 1752 | int num=vec_size(&sbytes); 1753 | if (num>=i) return vec_at(&sbytes,num-(i)); 1754 | else return 0; 1755 | } 1756 | void Free(){ 1757 | vec_free(&awords); 1758 | vec_free(&sbytes); 1759 | } 1760 | }; 1761 | 1762 | inline U32 hash(U32 a, U32 b, U32 c=0xffffffff) { 1763 | U32 h=a*110002499u+b*30005491u+c*50004239u; 1764 | return h^h>>9^a>>3^b>>3^c>>4; 1765 | } 1766 | 1767 | inline int charSwap(int c){ 1768 | if (c>='{' && c<127) c+='P'-'{'; 1769 | else if (c>='P' && c<'T') c-='P'-'{'; 1770 | else if ( (c>=':' && c<='?') || (c>='J' && c<='O') ) c^=0x70; 1771 | if (c=='X' || c=='`') c^='X'^'`'; 1772 | return c; 1773 | } 1774 | 1775 | // Predictor 1776 | 1777 | static const U32 primes[14]={0, 257,251,241,239,233,229,227,223,211,199,197,193,191}; 1778 | static const U32 tri[4]={0,4,3,7}, trj[4]={0,6,6,12}; 1779 | 1780 | // Parameters 1781 | // These parameters were tuned befor version 1 and may be bad. 1782 | const U32 m_e[10]={8,8,8,1,1,1,1,1,1,0}; // mixer error 1783 | const U32 m_s[10]={194, 237, 204, 70, 54, 55,55, 70,55, 6};// mixer shift 1784 | const U32 m_m[10]={36, 69, 19, 34, 23, 24,24, 34,24,4};// mixer error mul 1785 | 1786 | const U32 c_r[27]= { 3, 4, 6, 4, 6, 6, 2, 3, 3, 3, 6, 4, 3, 4, 5, 6, 2, 6, 4, 4, 4, 4, 4, 4, 4, 4, 4}; // contextmap run mul 1787 | const U32 c_s[27]= {28, 26, 28, 31, 34, 31, 33, 33, 35, 35, 29, 32, 33, 34, 30, 36, 31, 32, 32, 32, 32, 32, 33, 32, 32, 32, 32}; // contextmap pr mul 1788 | const U32 c_s2[27]={12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 14, 12, 12, 12, 12}; // ... 1789 | const U32 c_s3[27]={43, 33, 34, 28, 34, 29, 32, 33, 37, 35, 33, 28, 31, 35, 28, 30, 33, 34, 32, 32, 32, 32, 32, 32, 32, 32, 32}; 1790 | const U32 c_s4[27]={ 9, 8, 9, 5, 8, 12, 15, 8, 8, 12, 10, 7, 7, 8, 8, 13, 13, 14, 8, 8, 12, 12, 12, 12, 12, 12, 12}; 1791 | 1792 | int e_l[8]={1830, 1997, 1973, 1851, 1897, 1690, 1998, 1842}; 1793 | 1794 | const int MAXLEN=62; // longest allowed match + 1 1795 | U32 t[14]; 1796 | 1797 | int c1,c2,c3; 1798 | U8 words,spaces,numbers; 1799 | U32 word0,word1,word2,word3,wshift,w4,w4r,w4br,x4,x5,ismatch,firstWord,number0,number1,numlen0,numlen1,mybenum; 1800 | U32 fqcxt=0; 1801 | U32 AH1=0,AH2=0x765BA55C; 1802 | U32 fails=0, failz=0, failcount=0; 1803 | int nl,col,fc,fc1,nl1; 1804 | U32 t1[0x100]; 1805 | U32 t2[0x10000]; 1806 | int wp[0x10000]; 1807 | 1808 | U32 oState, wtype, ttype; 1809 | U32 nState; 1810 | U32 oStatew4,nStatew4; 1811 | 1812 | int ord,ord2; 1813 | U8 buffer[0x1000000]; 1814 | enum {BMASK=0xffffff}; 1815 | int pos; 1816 | int pr; 1817 | 1818 | StateMap smA[3]; 1819 | SmallStationaryContextMap scmA[8]; 1820 | Mixer1 mxA[10]; 1821 | ContextMap cmC[27]; 1822 | APM apmA[6]; 1823 | RunContextMap rcmA[1]; 1824 | BracketContext brcxt; 1825 | BracketContext qocxt; 1826 | BracketContext fccxt; 1827 | ColumnContext colcxt; 1828 | WordsContext worcxt; 1829 | 1830 | void PredictorInit() { 1831 | nState=nStatew4=0xffffffff; 1832 | pr=2048; 1833 | smA[0].Init(1<<9,1023,&STA1[0][0]);//match 1834 | smA[1].Init(1<<19,1023,&STA1[0][0]);//match 1835 | smA[2].Init(1<<16,1023,&STA1[0][0]);//match 1836 | 1837 | for (int i=0;i<8;i++){ 1838 | scmA[i].Init(8); 1839 | } 1840 | 1841 | mxA[1].Init(2*256,m_s[1],m_e[1],m_m[1]); 1842 | mxA[2].Init(6*256,m_s[2],m_e[2],m_m[2]); 1843 | mxA[3].Init(6*256,m_s[3],m_e[3],m_m[3]); 1844 | mxA[4].Init(8*256,m_s[4],m_e[4],m_m[4]); 1845 | mxA[5].Init(6*256,m_s[5],m_e[5],m_m[5]); 1846 | mxA[6].Init(7*256*4,m_s[6],m_e[6],m_m[6]); 1847 | mxA[7].Init(8*256,m_s[7],m_e[7],m_m[7]); 1848 | mxA[8].Init(8*256,m_s[8],m_e[8],m_m[8]); 1849 | mxA[9].Init(8*7*2,m_s[9],m_e[9],m_m[9]); 1850 | 1851 | apmA[0].Init(256); 1852 | apmA[1].Init(0x8000*2); 1853 | apmA[2].Init(0x8000*2); 1854 | apmA[3].Init(0x20000*2); 1855 | apmA[4].Init(0x10000*2); 1856 | apmA[5].Init(0x10000*2); 1857 | rcmA[0].Init(1*4096*4096,6); 1858 | 1859 | x.mxInputs[0].ncount=410; 1860 | x.mxInputs[1].ncount=8; 1861 | 1862 | for (int j=0;j<2;j++) { 1863 | x.mxInputs[j].ncount=(x.mxInputs[j].ncount+15)&-16; 1864 | alloc1(x.mxInputs[j].n,x.mxInputs[j].ncount+32,x.mxInputs[j].ptr,32); 1865 | } 1866 | // Provide inputs array info to mixers 1867 | for (int i=1;i<9;i++) 1868 | mxA[i].setTxWx(x.mxInputs[0].ncount,&x.mxInputs[0].n[0]); 1869 | // Final mixer 1870 | mxA[9].setTxWx(x.mxInputs[1].ncount,&x.mxInputs[1].n[0]); 1871 | 1872 | cmC[0].Init( 32*4096*4096,3|(c_r[0]<<8)|(c_s[0]<<16)|(c_s2[0]<<24),c_s3[0],&STA5[0][0],c_s4[0],0xf0,1); 1873 | cmC[1].Init( 32*4096*4096,1|(c_r[1]<<8)|(c_s[1]<<16)|(c_s2[1]<<24),c_s3[1],&STA1[0][0],c_s4[1],0xf0,1); 1874 | cmC[2].Init( 32*4096*4096,1|(c_r[2]<<8)|(c_s[2]<<16)|(c_s2[2]<<24),c_s3[2],&STA1[0][0],c_s4[2],0xf0,1); 1875 | cmC[3].Init( 32*4096*4096,1|(c_r[3]<<8)|(c_s[3]<<16)|(c_s2[3]<<24),c_s3[3],&STA1[0][0],c_s4[3],0xf0,1); 1876 | cmC[4].Init( 32*4096*4096,2|(c_r[4]<<8)|(c_s[4]<<16)|(c_s2[4]<<24),c_s3[4],&STA1[0][0],c_s4[4],0xf0,1); 1877 | cmC[5].Init( 32*4096*4096,1|(c_r[5]<<8)|(c_s[5]<<16)|(c_s2[5]<<24),c_s3[5],&STA3[0][0],c_s4[5],0xf0,1);//mem 16-8 1878 | cmC[6].Init( 1*4096*4096,1|(c_r[6]<<8)|(c_s[6]<<16)|(c_s2[6]<<24),c_s3[6],&STA1[0][0],c_s4[6],0xf0,1); 1879 | cmC[7].Init( 2*4096*4096,1|(c_r[7]<<8)|(c_s[7]<<16)|(c_s2[7]<<24),c_s3[7],&STA5[0][0],c_s4[7],0xf0,1); 1880 | cmC[8].Init( 32*4096*4096,3|(c_r[8]<<8)|(c_s[8]<<16)|(c_s2[8]<<24),c_s3[8],&STA4[0][0],c_s4[8],0,1); 1881 | cmC[9].Init( 32*4096,2|(c_r[9]<<8)|(c_s[9]<<16)|(c_s2[9]<<24),c_s3[9],&STA1[0][0],c_s4[9],0xf0,0); 1882 | cmC[10].Init( 32*4096,3|(c_r[10]<<8)|(c_s[10]<<16)|(c_s2[10]<<24),c_s3[10],&STA2[0][0],c_s4[10],0,1); 1883 | cmC[11].Init( 32*4096,4|(c_r[11]<<8)|(c_s[11]<<16)|(c_s2[11]<<24),c_s3[11],&STA2[0][0],c_s4[11],0,1); 1884 | cmC[12].Init( 16*4096,5|(c_r[12]<<8)|(c_s[12]<<16)|(c_s2[12]<<24),c_s3[12],&STA2[0][0],c_s4[12],0,1); 1885 | cmC[13].Init( 16*4096,8|(c_r[13]<<8)|(c_s[13]<<16)|(c_s2[13]<<24),c_s3[13],&STA2[0][0],c_s4[13],0,1);//+ 1886 | cmC[14].Init( 64*2*4096,3|(c_r[14]<<8)|(c_s[14]<<16)|(c_s2[14]<<24),c_s3[14],&STA5[0][0],c_s4[14],0xf0,0); 1887 | cmC[15].Init( 2*4096,2|(c_r[15]<<8)|(c_s[15]<<16)|(c_s2[15]<<24),c_s3[15],&STA2[0][0],c_s4[15],0xf0,0); 1888 | cmC[16].Init( 128*4096,2|(c_r[16]<<8)|(c_s[16]<<16)|(c_s2[16]<<24),c_s3[16],&STA1[0][0],c_s4[16],0,0); 1889 | cmC[17].Init( 4*4096*4096,3|(c_r[17]<<8)|(c_s[17]<<16)|(c_s2[17]<<24),c_s3[17],&STA1[0][0],c_s4[17],0xf0,1); 1890 | cmC[18].Init(32*4096*4096,6|(c_r[18]<<8)|(c_s[18]<<16)|(c_s2[18]<<24),c_s3[18],&STA5[0][0],c_s4[18],0xf0,1); 1891 | cmC[19].Init(32*4096*4096,5|(c_r[19]<<8)|(c_s[19]<<16)|(c_s2[19]<<24),c_s3[19],&STA5[0][0],c_s4[19],0xf0,1); 1892 | cmC[20].Init(32*4096*4096,2|(c_r[20]<<8)|(c_s[20]<<16)|(c_s2[20]<<24),c_s3[20],&STA1[0][0],c_s4[20],0xf0,1); 1893 | cmC[21].Init(32*4096*4096,2|(c_r[21]<<8)|(c_s[21]<<16)|(c_s2[21]<<24),c_s3[21],&STA3[0][0],c_s4[21],0xf0,1); 1894 | cmC[22].Init( 32*4096,2|(c_r[22]<<8)|(c_s[22]<<16)|(c_s2[22]<<24),c_s3[22],&STA2[0][0],c_s4[22],0x00,1);//++ 1895 | cmC[23].Init(16*4096*4096/2,1|(c_r[23]<<8)|(c_s[23]<<16)|(c_s2[23]<<24),c_s3[23],&STA3[0][0],c_s4[23],0xf0,1); 1896 | cmC[24].Init( 8*64*4096,1|(c_r[24]<<8)|(c_s[24]<<16)|(c_s2[24]<<24),c_s3[24],&STA1[0][0],c_s4[24],0,0); 1897 | cmC[25].Init( 512*4096,1|(c_r[25]<<8)|(c_s[25]<<16)|(c_s2[25]<<24),c_s3[25],&STA1[0][0],c_s4[25],0xf0,1); 1898 | cmC[26].Init( 512*4096,1|(c_r[26]<<8)|(c_s[26]<<16)|(c_s2[26]<<24),c_s3[26],&STA1[0][0],c_s4[26],0xf0,1); 1899 | 1900 | #if defined(TEXTMODE) 1901 | brcxt.Init(&brackets[0],8,false,512); 1902 | #else 1903 | brcxt.Init(&brackets[0],8); 1904 | #endif 1905 | qocxt.Init("es[0],4,true); 1906 | #if defined(TEXTMODE) 1907 | fccxt.Init(&fchar[0],20,false,512*2); 1908 | #else 1909 | fccxt.Init(&fchar[0],20); 1910 | #endif 1911 | colcxt.Init(); 1912 | worcxt.Init(); 1913 | } 1914 | 1915 | void PredictorFree(){ 1916 | smA[0].Free(); 1917 | for (int i=0;i<8;i++) scmA[i].Free(); 1918 | for (int i=1;i<10;i++) mxA[i].Free(); 1919 | for (int i=0;i<27;i++) cmC[i].Free(); 1920 | for (int i=0;i<6;i++) apmA[i].Free(); 1921 | rcmA[1].Free(); 1922 | free(x.mxInputs[0].ptr); 1923 | free(x.mxInputs[1].ptr); 1924 | brcxt.Free(); 1925 | qocxt.Free(); 1926 | fccxt.Free(); 1927 | colcxt.Free(); 1928 | worcxt.Free(); 1929 | } 1930 | 1931 | int buf(int i){ 1932 | return buffer[(pos-i)&BMASK]; 1933 | } 1934 | int bufr(int i){ 1935 | return buffer[i&BMASK]; 1936 | } 1937 | 1938 | //---------------- Match model 2--------------------- 1939 | // based on paq8px v208 1940 | struct HashElementForMatchPositions { // sizeof(HashElementForMatchPositions) = 3*4 = 12 1941 | #define mHashN 3 1942 | U32 matchPositions[mHashN]; 1943 | void Add(int pos) { 1944 | if (mHashN > 1) { 1945 | memmove(&matchPositions[1], &matchPositions[0], (mHashN - 1) * sizeof(matchPositions[0])); 1946 | } 1947 | matchPositions[0] = pos; 1948 | } 1949 | }; 1950 | 1951 | const int MINLEN_RM = 3; //minimum length in recovery mode before we "fully recover" 1952 | const int LEN1 = 5; // order x 1953 | const int LEN2 = 7; // 1954 | const int LEN3 = 9; 1955 | 1956 | struct MatchInfo { 1957 | U32 length; // rebased length of match (length=1 represents the smallest accepted match length), or 0 if no match 1958 | U32 index; // points to next byte of match in buf, 0 when there is no match 1959 | U32 lengthBak; // allows match recovery after a 1-byte mismatch 1960 | U32 indexBak; 1961 | U8 expectedByte; // prediction is based on this byte (buf[index]), valid only when length>0 1962 | bool delta; // indicates that a match has just failed (delta mode) 1963 | void Init(){ 1964 | length=0; 1965 | index=0; 1966 | lengthBak=0; 1967 | indexBak=0; 1968 | expectedByte=0; 1969 | delta=false; 1970 | } 1971 | bool isInNoMatchMode() const { 1972 | return length == 0 && !delta && lengthBak == 0; 1973 | } 1974 | 1975 | bool isInPreRecoveryMode() const { 1976 | return length == 0 && !delta && lengthBak != 0; 1977 | } 1978 | 1979 | bool isInRecoveryMode() const { 1980 | return length != 0 && lengthBak != 0; 1981 | } 1982 | 1983 | U32 recoveryModePos() const { 1984 | assert(isInRecoveryMode()); //must be in recovery mode 1985 | return length - lengthBak; 1986 | } 1987 | 1988 | U32 prio() { 1989 | return 1990 | (length != 0) << 31 | //normal mode (match) 1991 | (delta) << 30 | //delta mode 1992 | (delta ? (lengthBak>>1) : (length>>1)) << 24 | //the longer wins, halve 1993 | (index&0x00ffffff); //the more recent wins 1994 | } 1995 | bool isBetterThan(MatchInfo* other) { 1996 | return this->prio() > other->prio(); 1997 | } 1998 | 1999 | void update() { 2000 | //printf("- pos %d %d index %d length %d lengthBak %d delta %d\n", x.blpos, x.bpos, index, length, lengthBak, delta ? 1 : 0); 2001 | if (length != 0) { 2002 | const int expectedBit = (expectedByte >> ((8 - x.bpos) & 7)) & 1; 2003 | if (x.y != expectedBit) { 2004 | if (isInRecoveryMode()) { // another mismatch in recovery mode -> give up 2005 | lengthBak = 0; 2006 | indexBak = 0; 2007 | } 2008 | else { //backup match information: maybe we can recover it just after this mismatch 2009 | lengthBak = length; 2010 | indexBak = index; 2011 | delta = true; //enter into delta mode - for the remaining bits in this byte length will be 0; we will exit delta mode and enter into recovery mode on bpos==0 2012 | } 2013 | length = 0; 2014 | } 2015 | } 2016 | 2017 | if (x.bpos == 0) { 2018 | // recover match after a 1-byte mismatch 2019 | if (isInPreRecoveryMode()) { // just exited delta mode, so we have a backup 2020 | //the match failed 2 bytes ago, we must increase indexBak by 2: 2021 | indexBak++; 2022 | if (lengthBak < MAXLEN) { 2023 | lengthBak++; 2024 | } 2025 | if (bufr(indexBak) == c1) { // match continues -> recover 2026 | length = lengthBak; 2027 | index = indexBak; 2028 | } 2029 | else { // still mismatch 2030 | lengthBak = indexBak = 0; // purge backup (give up) 2031 | } 2032 | } 2033 | // extend current match 2034 | if (length != 0) { 2035 | index++; 2036 | if (length < MAXLEN) { 2037 | length++; 2038 | } 2039 | if (isInRecoveryMode() && recoveryModePos() >= MINLEN_RM) { // recovery seems to be successful and stable -> exit recovery mode 2040 | lengthBak = indexBak = 0; // purge backup 2041 | } 2042 | } 2043 | delta = false; 2044 | } 2045 | //printf(" pos %d %d index %d length %d lengthBak %d delta %d\n", x.blpos, x.bpos, index, length, lengthBak, delta ? 1 : 0); 2046 | } 2047 | 2048 | void registerMatch(const U32 pos, const U32 LEN) { 2049 | assert(pos != 0); 2050 | length = LEN - LEN1 + 1; // rebase 2051 | index = pos; 2052 | lengthBak = indexBak = 0; 2053 | expectedByte = 0; 2054 | delta = false; 2055 | } 2056 | }; 2057 | 2058 | const int matchN=4; // maximum number of match candidates 2059 | MatchInfo matchCandidates[matchN]; 2060 | U32 numberOfActiveCandidates=0; 2061 | HashElementForMatchPositions *mhashtable,*mhptr; 2062 | U32 mhashtablemask; 2063 | const int nST=3; 2064 | U32 ctx[nST]; 2065 | 2066 | bool isMatch(const U32 pos, const int MINLEN) { 2067 | for (int length = 1; length <= MINLEN; length++) { 2068 | if (buf(length) != bufr(pos - length)) 2069 | return false; 2070 | } 2071 | return true; 2072 | } 2073 | 2074 | void AddCandidates(HashElementForMatchPositions* matches, U32 LEN) { 2075 | U32 i = 0; 2076 | while (numberOfActiveCandidates < matchN && i < mHashN) { 2077 | U32 matchpos = matches->matchPositions[i]; 2078 | if (matchpos == 0) 2079 | break; 2080 | if (isMatch(matchpos, LEN)) { 2081 | bool isSame = false; 2082 | //is this position already registered? 2083 | for (U32 j = 0; j < numberOfActiveCandidates; j++) { 2084 | MatchInfo* oldcandidate = &matchCandidates[j]; 2085 | isSame = (oldcandidate->index == matchpos); 2086 | if (isSame) 2087 | break; 2088 | } 2089 | if (!isSame) { //don't register an already registered sequence 2090 | matchCandidates[numberOfActiveCandidates].registerMatch(matchpos, LEN); 2091 | numberOfActiveCandidates++; 2092 | } 2093 | } 2094 | i++; 2095 | } 2096 | } 2097 | 2098 | void MatchModel2update() { 2099 | //update active candidates, remove dead candidates 2100 | U32 n = max(numberOfActiveCandidates, 1); 2101 | for (U32 i = 0; i < n; i++) { 2102 | MatchInfo* matchInfo = &matchCandidates[i]; 2103 | matchInfo->update(); 2104 | if (numberOfActiveCandidates != 0 && matchInfo->isInNoMatchMode()) { 2105 | numberOfActiveCandidates--; 2106 | if (numberOfActiveCandidates == i) 2107 | break; 2108 | memmove(&matchCandidates[i], &matchCandidates[i + 1], (numberOfActiveCandidates - i) * sizeof(MatchInfo)); 2109 | i--; 2110 | } 2111 | } 2112 | 2113 | if( x.bpos == 0 ) { 2114 | U32 hash; 2115 | HashElementForMatchPositions* matches; 2116 | 2117 | hash = t[LEN3]; 2118 | matches = &mhashtable[(hash& mhashtablemask)]; 2119 | if (numberOfActiveCandidates < matchN) 2120 | AddCandidates(matches, LEN3); //longest 2121 | matches->Add(pos); 2122 | 2123 | hash = t[LEN2]; 2124 | matches = &mhashtable[(hash& mhashtablemask)]; 2125 | if (numberOfActiveCandidates < matchN) 2126 | AddCandidates(matches, LEN2); //middle 2127 | matches->Add(pos); 2128 | 2129 | hash = t[LEN1]; 2130 | matches = &mhashtable[(hash& mhashtablemask)]; 2131 | if (numberOfActiveCandidates < matchN) 2132 | AddCandidates(matches, LEN1); //shortest 2133 | matches->Add(pos); 2134 | 2135 | for (U32 i = 0; i < numberOfActiveCandidates; i++) { 2136 | matchCandidates[i].expectedByte = bufr(matchCandidates[i].index); 2137 | } 2138 | } 2139 | } 2140 | 2141 | int MatchModel2mix(int m) { 2142 | MatchModel2update(); 2143 | 2144 | for( int i = 0; i < nST; i++ ) { // reset contexts 2145 | ctx[i] = 0; 2146 | } 2147 | 2148 | int bestCandidateIdx = 0; //default item is the first candidate, let's see if any other candidate is better 2149 | for (U32 i = 1; i < numberOfActiveCandidates; i++) { 2150 | if (matchCandidates[i].isBetterThan(&matchCandidates[bestCandidateIdx])) 2151 | bestCandidateIdx = i; 2152 | } 2153 | 2154 | const U32 length = matchCandidates[bestCandidateIdx].length; 2155 | const U8 expectedByte = matchCandidates[bestCandidateIdx].expectedByte; 2156 | const bool isInDeltaMode = matchCandidates[bestCandidateIdx].delta; 2157 | const int expectedBit = length != 0 ? (expectedByte >> (7 - x.bpos)) & 1 : 0; 2158 | 2159 | U32 denselength = 0; // 0..27 2160 | if (length != 0) { 2161 | if (length <= 16) { 2162 | denselength = length - 1; // 0..15 2163 | } else { 2164 | denselength = 12 + ((length ) >> 2); // 16..27 2165 | } 2166 | ctx[0] = (denselength << 4) | (expectedBit << 3) | x.bpos; // 1..28*2*8 2167 | ctx[1] = ((expectedByte << 11) | (x.bpos << 8) | c1) ;//+ 1; 2168 | const int sign = 2 * expectedBit - 1; 2169 | x.mxInputs[m].add(sign * (length << 5)); 2170 | } else { // no match at all or delta mode 2171 | x.mxInputs[m].add(0); 2172 | } 2173 | 2174 | if( isInDeltaMode ) { // delta mode: helps predicting the remaining bits of a character when a mismatch occurs 2175 | ctx[2] = (expectedByte << 8) | x.c0; 2176 | } 2177 | 2178 | for( int i = 0; i < nST; i++ ) { 2179 | const U32 c = ctx[i]; 2180 | if( c != 0 ) { 2181 | smA[i].set(c,x.y); 2182 | const int p1 = smA[i].pr; 2183 | const int st = stretch(p1); 2184 | x.mxInputs[m].add(st >> 2); 2185 | x.mxInputs[m].add((p1 - 2048) >> 3); 2186 | } else { 2187 | x.mxInputs[m].add(0); 2188 | x.mxInputs[m].add(0); 2189 | } 2190 | } 2191 | return length; 2192 | } 2193 | 2194 | const U8 fcy[128]={ 2195 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2196 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2197 | 0, 0, 5, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 2198 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2199 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 2200 | 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2201 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2202 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 2203 | }; 2204 | 2205 | const U8 fcq[128]={ 2206 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2207 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2208 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2209 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2210 | 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 4, 5, 0, 0, 2211 | 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2212 | 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2213 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 2214 | }; 2215 | 2216 | U32 ttype1=0,wtype1=0,w41=0,w42=0; 2217 | int modelPrediction(int c0,int bpos,int c4){ 2218 | int i,c; 2219 | U32 h,j; 2220 | 2221 | if (bpos== 0){ 2222 | wshift=0; 2223 | c3=c2; 2224 | c2=c1; 2225 | c1=c4&0xff; 2226 | 2227 | // if TEXTMODE swap cars 2228 | #ifdef TEXTMODE 2229 | i=wrt_w[charSwap(c1)]; 2230 | #else 2231 | i=wrt_w[c1]; 2232 | #endif 2233 | nStatew4=i; 2234 | w4=w4*4+i; 2235 | buffer[pos&BMASK]=c1; 2236 | pos++; 2237 | // Column content update 2238 | #ifndef TEXTMODE 2239 | if (colcxt.lastfc()==GREATERTHAN) colcxt.nlChar=GREATERTHAN; 2240 | if (colcxt.lastfc()==SQUAREOPEN && colcxt.nlChar==GREATERTHAN) colcxt.nlChar=10; 2241 | #endif 2242 | colcxt.Update(c1,c4&0xffffff); 2243 | // Bracket content update 2244 | #ifdef TEXTMODE 2245 | if (c1<128)brcxt.Update( c1 ); 2246 | #else 2247 | if (c1<'a')brcxt.Update( c1 ); // advance bracket context only if no letters, so we do not get out of range 2248 | #endif 2249 | cmC[25].set((brcxt.context<<8)+c1); 2250 | // Quote content update 2251 | qocxt.Update(c1); 2252 | // Look for byte stream x4 and order X context end. 2253 | if ( c1!=SPACE) 2254 | if (c1=='$' || c1==SQUARECLOSE|| c1==VERTICALBAR|| c1==')'|| c1==SQUAREOPEN){ 2255 | if ( c1!=c2) 2256 | for (i=13; i>0; --i) // update order X context hashes 2257 | t[i]=t[i-1]*primes[i]; 2258 | // Duplicate input byte, marks end 2259 | x4=(x4<<8)+c2; 2260 | } 2261 | // Update byte stream x4 and order X contexts 2262 | x4=(x4<<8)+c1; 2263 | for (i=13; i>0; --i) 2264 | t[i]=t[i-1]*primes[i]+c1+i*256; 2265 | 2266 | for (i=3; i<6; ++i) 2267 | cmC[0].set(t[i]); 2268 | cmC[1].set(t[6]); 2269 | cmC[2].set(t[8]); 2270 | cmC[3].set(t[13]); 2271 | 2272 | #ifdef TEXTMODE 2273 | // if TEXTMODE swap cars 2274 | nState=wrt_t[charSwap(c1)]; 2275 | #else 2276 | nState=wrt_t[c1]; 2277 | #endif 2278 | words=words<<1; 2279 | spaces=spaces<<1; 2280 | numbers=numbers<<1; 2281 | j=c1; 2282 | #ifdef TEXTMODE 2283 | bool isUpper=false; 2284 | bool isLetter=false; 2285 | if ( ((j-'A') <= ('Z'-'A'))) j =j+32,isUpper=true; 2286 | #endif 2287 | if (((j-'a') <= ('z'-'a')) || (c1>127 && c2!=12)) { 2288 | if (word0==0){ 2289 | if (c2==ATSIGN || c2==7)worcxt.Set(c3); else worcxt.Set(c2); 2290 | } 2291 | // if (word0 && c1<='z' && c2>127)worcxt.Update(word0,c1),worcxt.Set(15); 2292 | #ifdef TEXTMODE 2293 | isLetter=true; 2294 | #endif 2295 | words=words|1; 2296 | word0=word0*2104+j; //263*8 2297 | //if ((words&5)==5 && c2=='-') c2=SPACE,spaces=spaces|2,w4=(w4&0xFFFFFFF3)+wrt_w[SPACE]*4; 2298 | // ' or 0x27 is used for quotes, sometimes it has other meaning 2299 | // remove quote content if any of the fallowing is true 2300 | const int word3bit=(words&7); 2301 | if (((word3bit==5) && (c2==APOSTROPHE))|| // "x'x" where x is any letter in word 2302 | ((word3bit==1) && (c3==SQUARECLOSE) && (c2==APOSTROPHE))|| // "]'x" where x is any letter in word 2303 | ((word3bit==1) && (numbers&4)&&(c2==APOSTROPHE)) // "y'x" where y is number and x is any letter in word //(c3>='0' && c3<='9') 2304 | )qocxt.Update(qocxt.cxt); 2305 | } else { 2306 | // ' or 0x27 is used for quotes, sometimes it has other meaning 2307 | // remove quote content if any of the fallowing is true 2308 | const int word3bit=(words&7); 2309 | if ( ((word3bit==4)&& (c1==SPACE) && (c2==APOSTROPHE))|| // "x' " where x is any letter in word 2310 | ((c1==ATSIGN) && (numbers&4) && (c2==APOSTROPHE)) || // "x'@" where x is number // this is somehow semi good-bad //(c3>='0' && c3<='9') 2311 | ((word3bit==4) && (c1==ATSIGN) && (c2==APOSTROPHE)) // "x'@" where x is any letter in word // this is somehow semi good-bad 2312 | )qocxt.Update(qocxt.cxt); 2313 | if (word0){ 2314 | word3=word2*47; 2315 | word2=word1*53; 2316 | word1=word0*83; 2317 | // Update word content 2318 | worcxt.Update(word0,c1); 2319 | if(firstWord==0) { 2320 | firstWord=word0; 2321 | } 2322 | w42=w41; 2323 | ttype1=wtype1=w41=0; 2324 | } 2325 | wp[word0&0xffff]=pos; 2326 | word0=0; 2327 | if ((c1>='0' && c1<='9') ) { 2328 | numbers=numbers+1; 2329 | if(numbers&4 && c2==',') number0=number1,number1=0,numlen0=numlen1,numlen1=0; 2330 | if (mybenum && numlen1<=2) number0=number1,number1=0,numlen0=numlen1,numlen1=0; 2331 | number0=number0*10+(c1&0x0f); 2332 | numlen0=min(19,numlen0+1);mybenum=0; 2333 | }else{ 2334 | 2335 | if (numlen0 ||((numbers&0xf)==0)){ 2336 | number1=number0,numlen1=numlen0,number0=numlen0=0; 2337 | } 2338 | if (numlen1<=2 &&numlen1&&((numbers&5)==5) && numlen0==0 && c2=='.') mybenum=2; 2339 | else if (numlen1<=2&&numlen1&&(numbers&2) && numlen0==0 && c1=='.') mybenum=1; 2340 | else if (mybenum==1 && c1!='.') mybenum=0; 2341 | } 2342 | 2343 | if (c1==SPACE) { 2344 | spaces++; 2345 | } 2346 | else if (c1==10 ) { 2347 | fc=fc1=firstWord=0; 2348 | nl1=nl; 2349 | nl=pos-1; 2350 | wtype=(wtype<<7); 2351 | w4=w4|0x3fc; 2352 | words=0xfc; 2353 | worcxt.Reset(); 2354 | w4r= w4r<<2; 2355 | } 2356 | else if ((c1=='.') || c1==')' || c1==QUESTION) { 2357 | wtype= wtype<<7; 2358 | ttype= ttype<<7; 2359 | words= words|0xfe; 2360 | x5=(x5<<8)+(c4&0xff); 2361 | w4=w4|204; 2362 | w4r= w4r&0xffffffc0; 2363 | } 2364 | else if (c1==',') { 2365 | words=words|0xfc; 2366 | } 2367 | else if (c1==COLON) { 2368 | ttype= (ttype&0xfffffff8)+4; 2369 | w4=w4|12;// 1100 2370 | x5=(x5<<8)+(c4&0xff); 2371 | } 2372 | else if (c1==CURLYCLOSE || c1==CURLYOPENING) { 2373 | words = words | 0xfc; 2374 | wtype= wtype&0xffffffc0; 2375 | x5=(x5<<8)+(c4&0xff); 2376 | ttype= (ttype&0xfffffff8)+3; 2377 | } 2378 | else if (c1==SQUARECLOSE) { 2379 | ttype= (ttype&0xfffffff8)+3; 2380 | } 2381 | else if (c1==LESSTHAN || c2=='&') { 2382 | words=words|0xfc; 2383 | } 2384 | else if (c1==EQUALS) { 2385 | ttype=(ttype&0xfffffff8)+4; 2386 | c2='.'; // ok 2387 | } 2388 | else if (c1==SEMICOLON) { 2389 | worcxt.Reset(); 2390 | } 2391 | if (c1=='!' && c2=='&') {// ' ' to '&!' to ' ' 2392 | c1=SPACE; 2393 | c4=(c4&0xffffff00)+SPACE; 2394 | w4=(w4&0xfffffffc)+wrt_w[SPACE]; 2395 | ttype=(ttype&0xfffffff8)+wrt_t[SPACE]; 2396 | } 2397 | if (c1=='.' ) wshift=1,worcxt.Reset(); 2398 | } 2399 | 2400 | x5=(x5<<8)+(c4&0xff); 2401 | // Switch state if it is new 2402 | if (oStatew4!=nStatew4){ 2403 | w4r=(w4r<<2)+nStatew4; 2404 | oStatew4=nStatew4; 2405 | 2406 | } 2407 | // Switch state if it is new 2408 | if (oState!=nState){ 2409 | wtype=(wtype<<3)+nState; 2410 | w41=(w41<<3)+7; 2411 | w42=(w42<<3)+7; 2412 | oState=nState; 2413 | } 2414 | wtype1=(wtype1<<2)+3; 2415 | ttype=(ttype<<3)+nState; 2416 | ttype1=(ttype1<<3)+7; 2417 | U8 brcontext=brcxt.cxt; 2418 | 2419 | rcmA[0].set(word3*53+c1+193 * (ttype & 0x7fff),c1); 2420 | //Retrun index of bracket or quote in array, if found add 1 to result. value is in range 1-7, 0 if not found 2421 | w4br=0; 2422 | if(brcxt.context)w4br=fcy[brcontext];//FindQy(brcontext)+1; 2423 | if(brcxt.context==0 &&qocxt.context)w4br=fcy[qocxt.context>>8];//FindQy(qocxt.context>>8)+1; 2424 | // Column and first char 2425 | col=colcxt.collen(); 2426 | int above=buffer[(nl1+col)&BMASK]; 2427 | int above1=buffer[(nl1+col-1)&BMASK]; 2428 | #ifndef TEXTMODE 2429 | // Filtered wiki. We ignore 10 as new line char, '>' marks new line. 2430 | if (colcxt.nlChar==GREATERTHAN) { 2431 | above=colcxt.colb(1,0); 2432 | above1=colcxt.colb(1,1); 2433 | } 2434 | #endif 2435 | if (colcxt.isNewLine()) { 2436 | // Reset contexts when there are two empty lines 2437 | if((colcxt.nlpos(0)+2-colcxt.nlpos(1))< 4){ 2438 | fccxt.Reset(); 2439 | brcxt.Reset(); 2440 | qocxt.Reset(); 2441 | //dotcxt.Reset(); 2442 | } 2443 | fc=colcxt.lastfc(); 2444 | #ifdef TEXTMODE 2445 | if (isUpper==true && isLetter==true) fc=ATSIGN; 2446 | #else 2447 | // Reset first char context when there is >. For filtered wiki. 2448 | if(fc==GREATERTHAN) fccxt.Reset();//? 2449 | #endif 2450 | // Set paragraph 2451 | if (fc==ATSIGN) fc1=1; 2452 | else fc1=0; 2453 | // Set new first char 2454 | fccxt.Update(fc); 2455 | } 2456 | 2457 | #ifdef TEXTMODE 2458 | if ( (col>2 && c1>ATSIGN) || (col>2 && !(c1>='a' && c1<='z')) ) { 2459 | #else 2460 | if (col>2 && c1>ATSIGN ){ 2461 | #endif 2462 | // Before updating first char context look: 2463 | // If link or template ended then remove any vertical bars |. [xx|xx] {xx|xx} 2464 | if (fccxt.cxt==VERTICALBAR && (c1==SQUARECLOSE || c1==CURLYCLOSE)) while( fccxt.cxt==VERTICALBAR) fccxt.Update(10); 2465 | // If html link ends with space or ] 2466 | if ((fccxt.cxt==COLON || fccxt.cxt==31 ) && (c1==SPACE || c1==SQUARECLOSE)) while( fccxt.cxt==COLON || fccxt.cxt==31) fccxt.Update(10); 2467 | #ifndef TEXTMODE 2468 | if ( c1<128 ) 2469 | #endif 2470 | fccxt.Update(c1); 2471 | } 2472 | // Switch from possible category link to http link 2473 | if (fccxt.cxt==COLON && c2=='/' && c1=='/') fccxt.Update(10),fccxt.Update(31); 2474 | // Wiki link in the beginning of line 2475 | if (colcxt.lastfc(0)==SQUAREOPEN && c1==SPACE) { 2476 | if(c2==SQUARECLOSE || c3==SQUARECLOSE) { 2477 | fc=ATSIGN; 2478 | fc1=1; // Paragraph 2479 | // Line started with '[' and last chars were '] ', so probably rest of the line/paragraph fallows. 2480 | // Reset first char context and set new first char as paragraph start and continue. 2481 | // Problem: inlink links also reset, like images that have description etc. 2482 | fccxt.Reset(); 2483 | fccxt.Update(fc); 2484 | } 2485 | } 2486 | // Fist char was space, look for another non-space char 2487 | if (fc==SPACE && c1!=SPACE) { 2488 | fc=min(c1,96); 2489 | #ifdef TEXTMODE 2490 | if (isUpper==true && isLetter==true) fc=ATSIGN; 2491 | #endif 2492 | // Paragraph 2493 | if (fc==ATSIGN) fc1=1; 2494 | else fc1=0; 2495 | // Set new first char, we keep space from previous update 2496 | fccxt.Update(fc); 2497 | } 2498 | const U8 fccontext =fccxt.cxt; 2499 | if(w4br==0 &&fccxt.context)w4br=fcy[fccontext];//FindQy(fccontext)+1; 2500 | fqcxt=fcq[fccontext];//FindQy2(fccontext); 2501 | 2502 | cmC[26].set((fccxt.context&0xff00)+c1+(w4&12)*256+((brcontext+ brcxt.last())<<24)); 2503 | // List - needs fixme 2504 | if (fc=='*' && c1!=SPACE) { 2505 | fc=min(c1,96); 2506 | } 2507 | if (fc=='&' && c1==LESSTHAN) fc=30; // Inwiki html 2508 | // We have bold/italic first char/words. When words surrounded by ' end there is problably rest of paraghraph. 2509 | // Reset first char context and set new first char as paragraph start and continue. 2510 | if (colcxt.lastfc(0)==APOSTROPHE && (c1==SPACE)) { 2511 | if(c2==APOSTROPHE || c3==APOSTROPHE ){ 2512 | fc=ATSIGN; 2513 | fc1=1; // Paragraph 2514 | fccxt.Reset(); // Not really needed 2515 | fccxt.Update(fc); // 2516 | } 2517 | } 2518 | if ((fc!=ATSIGN) && ((c4&0xffffff)==0x4a2f2f)) {//http link - fixme 2519 | fc=31; 2520 | } 2521 | // Words surrounded by () 2522 | if ((worcxt.sBytes(1)&0xff)==')' ){ 2523 | bool isC=false; 2524 | for (int i=1; i<8; i++) if ((worcxt.sBytes(i)>>8)=='(') isC=true; 2525 | if (isC){ 2526 | while( (worcxt.sBytes(1)>>8)!='(' ) worcxt.Remove(); 2527 | worcxt.Remove(); 2528 | } 2529 | } 2530 | // Words surrounded by [| as wiki internal link: word [word word|word] word. 2531 | if ((worcxt.sBytes(1)&0xff)==VERTICALBAR ){ 2532 | bool isC=false; 2533 | for (int i=1; i<8; i++) if ((worcxt.sBytes(i)>>8)==SQUAREOPEN) isC=true; 2534 | if (isC){ 2535 | while( (worcxt.sBytes(1)>>8)!=SQUAREOPEN ) worcxt.Remove(); 2536 | worcxt.Remove(); 2537 | } 2538 | } 2539 | // Template - tiny gain 2540 | if (fccontext==VERTICALBAR && colcxt.isTemp==true){ 2541 | bool isC=false; 2542 | for (int i=1; i<10; i++) if ((worcxt.sBytes(i)&0xff)==EQUALS) isC=true; 2543 | if (isC){ 2544 | while( (worcxt.sBytes(1)&0xff)!=EQUALS ) worcxt.Remove(); 2545 | worcxt.Remove(); 2546 | } 2547 | } 2548 | 2549 | if (word0) { 2550 | h=word0*271+(c4&0xff); 2551 | } else { 2552 | h=word0*271+c1; 2553 | } 2554 | // Contexts 2555 | // Word stream cm(4-5) 2556 | if (c1==12)cmC[4].set(0);else cmC[4].set(word0+(number0*191+numlen0)); 2557 | if (c1==12)cmC[4].set(0); 2558 | else cmC[4].set(h+word1); 2559 | 2560 | cmC[5].set(h+ word2*71); 2561 | 2562 | cmC[6].set(((ttype&0x3f)<<16)+(c4&0xffff)); 2563 | 2564 | cmC[8].set((c4 & 0xffffff) + ((w4 << 18) & 0xff000000)); // fcx=31 bad 2565 | cmC[8].set((wtype&0x3fffffff)*4+(w4&3)); 2566 | cmC[8].set((fccontext*4) + ((wtype & 0x3ffff) << 9 )+w4br); 2567 | 2568 | cmC[9].set(colcxt.lastfc(0) | (fccontext<< 15) | ((ttype & 63) << 7)|(brcontext << 24) ); 2569 | cmC[9].set((colcxt.lastfc(0) | ((c4 & 0xffffff) << 8))); 2570 | 2571 | cmC[10].set( (w4 & 3) +word0*11); 2572 | cmC[10].set(c4 & 0xffff); 2573 | cmC[10].set(((fc << 11) | c1)+((w4 & 3)<< 18)); 2574 | 2575 | cmC[11].set((w4 & 15)+((ttype & 7) << 6 )); 2576 | cmC[11].set(c1 | ((col * (c1 == SPACE)) << 8)|((w4 & 15) << 16)); 2577 | 2578 | cmC[11].set(fc1?firstWord:(fc<< 11)); 2579 | if (c1==12 )cmC[11].set(0); 2580 | else cmC[11].set((91 * 83* worcxt.Word(1) + 89 * word0)); 2581 | 2582 | cmC[12].set((c1 + ((ttype & 0x38) << 6))); 2583 | cmC[12].set(worcxt.fword*11+w4br); 2584 | cmC[12].set(c1+word0+number0*191 ); 2585 | cmC[12].set(((c4 & 0xffff) << 16) | (fccontext << 8) |fc); 2586 | cmC[12].set(((wtype & 0xfff)<< 8)+((w4 & 0xfc))); 2587 | 2588 | // Switch between word/paragraph or column mode 2589 | if (fc1==1){ 2590 | // Word 2591 | cmC[13].set(worcxt.fword*3191+(w4 & 3)); 2592 | cmC[13].set(h+firstWord*89); 2593 | cmC[13].set(word0*53+c1+w4br); 2594 | }else{ 2595 | // Column 2596 | cmC[13].set(above | ((ttype & 0x3f) << 9) | (colcxt.collen() << 19)| ((w4 & 3) << 16) ); 2597 | cmC[13].set(h+firstWord*89); 2598 | cmC[13].set(above | (c1 << 16)| ((col+numlen0+w4br) << 8)| (above1<< 24) ); 2599 | } 2600 | // List 2601 | if (colcxt.lastfc()=='*' ){ 2602 | cmC[13].set(( ( fccontext) << 8) | ((w4br & 0xfff) << 16)); 2603 | cmC[13].set(c1); 2604 | cmC[13].set(word0); 2605 | }else{ 2606 | // Table 2607 | cmC[13].set(wrt_w[bufr(colcxt.abovecellpos)]|( ( fccontext) << 8) | ((w4br & 0xff) << 16)); 2608 | cmC[13].set(bufr(colcxt.abovecellpos)|( ( c1) << 8) ); 2609 | cmC[13].set( word0+wrt_w[bufr(colcxt.abovecellpos)] ); 2610 | } 2611 | 2612 | cmC[14].set((x4 & 0xff00ff) ); 2613 | cmC[14].set((x4 & 0xff0000ff) | ((ttype & 0xe07) << 8)); 2614 | 2615 | // Indirect 2616 | U32 f=(c4>>8)&0xffff; 2617 | t2[f]=(t2[f]<<8)|c1; 2618 | f=c4&0xffff; 2619 | f=f|(t2[f]<<16); 2620 | U32 d=(c4>>8)&0xff; 2621 | t1[d]=(t1[d]<<8)|c1; 2622 | d=c1|(t1[c1]<<8); 2623 | 2624 | t1[brcontext]=(t1[brcontext]<<2)|(w4&3); // this is wierd, also end is bad 2625 | U32 d4=(ttype&7)|(t1[brcontext]<<3); 2626 | if (c1==12 || fccontext== CURLYOPENING ) cmC[7].set(0);else 2627 | cmC[7].set(d4); 2628 | 2629 | cmC[14].set((d4& 0xffff) | ((ttype & 0x38) << 16)); 2630 | // Two indirect bytes with current byte 2631 | cmC[13].set((f& 0xffffff)); 2632 | 2633 | cmC[15].set((c1 << 8) | (d >> 2)| (fc << 16)); // fixme 2634 | cmC[15].set((c4 & 0xffff)+(c2==c3?1:0)); 2635 | 2636 | cmC[16].set((ttype & ttype1)*256 | ((w4 &wtype1& 255) << 0)); 2637 | cmC[16].set(x4); 2638 | // Word stream. word(1) with first char context and last bit3word(1-5) 2639 | cmC[17].set(257 * word1+fccontext + 193 * (ttype & ttype1)); // fc==* problem 2640 | // First char, current byte and non repeating bit2word(1-6) 2641 | cmC[17].set(fc|((w4r & 0xfff) << 9) | ((c1 ) << 24));//end is good (lang) 2642 | if (colcxt.lastfc()== SQUAREOPEN && (fccontext==COLON)) 2643 | cmC[17].set( worcxt.fword*83+(w4 & 3)*11+brcontext); // all category/language/image links (better as standalone) 2644 | else 2645 | cmC[17].set((x4 & 0xffff00)+ brcontext+(fccontext<< 24)); 2646 | 2647 | cmC[18].set(d); 2648 | cmC[18].set(((d& 0xffff00)>>4) | ((w4 & 0xf) )| ((ttype & 0xfff) << 20)); 2649 | cmC[18].set((x4 >>16) | ((w4 & 255) << 24)); 2650 | #ifdef TEXTMODE 2651 | cmC[18].set((c1 << 11) | ((f & 0xffffff)>>16) ); 2652 | #else 2653 | if (c1>127)cmC[18].set(( (((w4 & 12)*256)+c1) << 11) | ((f & 0xffffff)>>16) ); 2654 | else cmC[18].set((c1 << 11)| (w4br << 8) | ((f & 0xffffff)>>16) ); 2655 | #endif 2656 | cmC[18].set((fccontext*4+w4br) | ((c4 & 0xffff)<< 9)| ((w4 & 0xff) << 24)); 2657 | cmC[18].set(((f >> 16) )| ((w4 & 0x3c)<< 25 )| (((ttype & 0x1ff))<< 16 )); 2658 | 2659 | cmC[19].set(((words) )+((( spaces ))<< 8)+((w4&15)<< 16)+(((wtype>>3)&511)<< 21)+(fc1<<30)); 2660 | cmC[19].set(c1 + ((ttype<< 5) & 0x1fffff00)); 2661 | #ifdef TEXTMODE 2662 | cmC[19].set(ttype); 2663 | #else 2664 | cmC[19].set(w4r*16+w4br ); 2665 | #endif 2666 | // Indirect byte with bracket context and last non repeating bit2words(2-10) 2667 | cmC[19].set(((d& 0xffff)>>8) + ((64 * w4r) & 0x3ffff00)+(brcontext<< 25)); // end good 2668 | // Byte from prvious word(0) pos if in range(255) with current byte and first char 2669 | int dd=pos-wp[word0&0xffff]; 2670 | if (dd>255) 2671 | dd=256 + (c1<<16); 2672 | else dd=dd + (buf(dd)<< 8)+(c1 << 16); 2673 | if (fccontext==64 && brcontext==SQUAREOPEN) cmC[19].set(0);else 2674 | cmC[19].set((dd )| (fc << 24)); 2675 | 2676 | // Byte stream of x4, msb of byte(4), 4 msb bits of byte(2,3) and full byte(1) 2677 | cmC[20].set((x4&0x80f00000)+((x4&0x0000f0ff) << 12) ); 2678 | // Paragraph or column. 2679 | // In Paragraph: disabled when escaped utf8 or html link 2680 | // In Column: when col is max(31) use last two bytes only otherwise add above bytes 2681 | if (fc1==1){ 2682 | // word 2683 | if (c1==12 || fccontext==31|| fccontext==CURLYOPENING ) 2684 | cmC[20].set(0); 2685 | else 2686 | cmC[20].set(h+worcxt.Word(1) *53 *79+worcxt.Word(3) *53*47 *71); 2687 | }else{ 2688 | //column 2689 | if (col==31) 2690 | cmC[20].set( c4 << 16); 2691 | else 2692 | cmC[20].set(above | ((c4 &0xffff)<< 16)| (above1<< 8)); 2693 | } 2694 | 2695 | 2696 | // Word/centence. Disabled when: escaped utf8, template (onliner) or table beginning, html link. 2697 | if (c1==12 || fccontext==CURLYOPENING|| fccontext==31 || fc==30 ||brcontext ==LESSTHAN ){ 2698 | 2699 | cmC[21].set(0);cmC[21].set(0);} 2700 | else{ 2701 | // Word/Centence with current word(0), word(1) and word(2) 2702 | cmC[21].set(worcxt.Word(1)*83*1471-word0*53+worcxt.Word(2)); 2703 | cmC[21].set(h+worcxt.Word(2) *53 *79+worcxt.Word(3) *53*47 *71); 2704 | } 2705 | // Last byte in wtype and w4 type with first char and bracket index 2706 | cmC[22].set(((wtype&7)<< 10) + (w4&3)+fc*4+ (w4br<< 24)); 2707 | // Current word or number 2708 | cmC[22].set( (word0*3301+number0*3191 ));//3191 2709 | if (c1==12 || fccontext==CURLYOPENING|| fccontext==31 || fc==30 ||brcontext ==LESSTHAN ) cmC[23].set(0); else //end fc [ ????? 2710 | // Word/centence and non-repeating bit3words upto word(2) with bracket/firstchar index 2711 | cmC[23].set(w4br+ worcxt.Word(2) * (wtype&w42 )); 2712 | 2713 | scmA[0].set(c1); 2714 | scmA[1].set(c2*(fc1)); 2715 | scmA[2].set((f&0xffffff)>>16); 2716 | scmA[3].set(ttype&0x3f); 2717 | scmA[4].set(w4&0xff); 2718 | scmA[5].set(brcontext); 2719 | scmA[6].set(fc1+ 2*((wtype&0x3f)) ); 2720 | scmA[7].set(fc); 2721 | 2722 | if (wshift||c1==10) { 2723 | word3=word3*47, word2=word2*53, word1=word1*83; 2724 | } 2725 | 2726 | cmC[24].set((w4br*256)+fc+(((wtype>>0)&0xFFF)<< 16)); 2727 | // Some APM context 2728 | AH1=hash((x5>>0)&255, (x5>>8)&255, (x5>>16)&0x80ff); 2729 | AH2=hash(19, x5&0x80ffff); 2730 | } 2731 | const int c0b=c0<<(8-bpos); 2732 | ismatch=MatchModel2mix(0); 2733 | 2734 | scmA[0].mix(0); 2735 | scmA[1].mix(0); 2736 | scmA[2].mix(0); 2737 | scmA[3].mix(0); 2738 | scmA[4].mix(0); 2739 | scmA[5].mix(0); 2740 | scmA[6].mix(0); 2741 | if (fccxt.cxt==COLON && fccxt.last()==SQUAREOPEN) { 2742 | x.mxInputs[0].add(0); 2743 | x.mxInputs[0].add(0); 2744 | } 2745 | else 2746 | scmA[7].mix(0); 2747 | 2748 | // order X 2749 | ord=cmC[0].mix(0); 2750 | if (ord==3) ord=2; // low max 2 2751 | ord=ord+cmC[1].mix(0); 2752 | ord=ord+cmC[2].mix(0); 2753 | ord=ord+cmC[3].mix(0); 2754 | if (c1==12) 2755 | cmC[4].mix4(0),cmC[4].mix4(0),cmC[4].cn=ord2=0; // For speed, also improves compression 2756 | else 2757 | ord2=cmC[4].mix(0); 2758 | if (c1==12) 2759 | cmC[5].mix4(0),cmC[5].cn=0; 2760 | else 2761 | ord2=ord2+cmC[5].mix(0); 2762 | cmC[6].mix(0); 2763 | cmC[7].mix(0); 2764 | cmC[8].mix(0); 2765 | cmC[9].mix(0); 2766 | cmC[10].mix(0); 2767 | cmC[11].mix(0); 2768 | cmC[12].mix(0); 2769 | if (c1==12) { 2770 | cmC[13].mix4(0); 2771 | cmC[13].mix4(0); 2772 | cmC[13].mix4(0); 2773 | cmC[13].mix4(0); 2774 | cmC[13].mix4(0); 2775 | cmC[13].mix4(0); 2776 | cmC[13].mix4(0); 2777 | cmC[13].cn=0; 2778 | }else{ 2779 | cmC[13].mix(0); 2780 | } 2781 | cmC[14].mix(0); 2782 | cmC[15].mix(0); 2783 | cmC[16].mix(0); 2784 | cmC[17].mix(0); 2785 | cmC[18].mix(0); 2786 | cmC[19].mix(0); 2787 | if (fc1==1 && (c1==12 || fccxt.cxt==31|| fccxt.cxt==CURLYOPENING ) ) 2788 | cmC[20].cn=1,cmC[20].mix(0), cmC[20].mix4(0); 2789 | else 2790 | cmC[20].mix(0); 2791 | // order Word 2792 | if (c1==12 || fccxt.cxt==CURLYOPENING|| fccxt.cxt==31 || fc==30 || brcxt.cxt==LESSTHAN ) 2793 | cmC[21].mix4(0),cmC[21].mix4(0),cmC[21].cn=0; // For speed, also improves compression 2794 | else 2795 | ord2=ord2+(cmC[21].mix(0)); 2796 | cmC[22].mix(0); 2797 | if (c1==12 || fccxt.cxt==CURLYOPENING|| fccxt.cxt==31 || fc==30 || brcxt.cxt ==LESSTHAN ) 2798 | cmC[23].mix4(0), cmC[23].cn=0; // For speed, also improves compression 2799 | else 2800 | ord2=ord2+cmC[23].mix(0); 2801 | cmC[24].mix(0); 2802 | cmC[25].mix(0); 2803 | cmC[26].mix(0); 2804 | rcmA[0].mix(0); 2805 | 2806 | // Mixer 2807 | 2808 | // reference 2809 | // if(bpos){ 2810 | // c=c0<<(8-bpos); if(bpos==1)c=c+c3/2; 2811 | // c=(min(bpos,5))*256+c1/32+8*(c2/32)+(c&192); 2812 | // } 2813 | // else c=c3/128+(c4>>31)*2+4*(c2/64)+(c1&240); 2814 | 2815 | // mixer 1 2816 | // at bpos=0 context is last 2 bit2word and 1 bit3word 2817 | // at bpos=1-3 context is last 2 bit2word and 1 bit3word of current bracket or quote 2818 | // at bpos=4-7 context is last 1 bit2word, current bit2word from c0 and bit3word of current bracket or quote 2819 | if (bpos==0) mxA[1].cxt=(w4&63)*8 + (ttype&7); 2820 | else if (bpos>3) { 2821 | c=wrt_w[c0b&255]; 2822 | mxA[1].cxt=(((w4<<2)&63)+c)*8+w4br; 2823 | } else 2824 | mxA[1].cxt=(w4&63)*8 +w4br; 2825 | 2826 | // mixer 2 2827 | // at bpos=0 context is was byte(3,4) a word and 2 bit3word 2828 | // at bpos=1 context is bit 1xxxxxxx from c0, was byte(2) a word, bit pos max 5,last 1 bit3word and 1 bit3word of current first char context 2829 | // at bpos=2 context is bit 11xxxxxx (bit pos 2) from c0, was byte(2) a word, bit pos max 5,last 1 bit3word and 1 bit3word of current first char context 2830 | // at bpos=3 context is bit 111xxxxx (bit pos 3) from c0, was byte(2) a word, bit pos max 5,last 1 bit3word and 1 bit3word of current first char context 2831 | // at bpos=4-7 context is bit current bit2word from c0, bit pos max 5,last 1 bit3word and 1 bit3word of current first char context 2832 | if (bpos){ 2833 | c=c0b; 2834 | if (bpos==1) c=c+16 * (words*2& 4); 2835 | else if (bpos>3) c=wrt_w[c0b&255]*64; 2836 | c=(min(bpos,5))*256+(wtype&7)+fqcxt*8+(c&192); //fqcxt -> cm(12,1) 2837 | } 2838 | else c=(words&12)*16+(wtype&7)+w4br*8; 2839 | mxA[2].cxt=c; 2840 | 2841 | // mixer 3 2842 | // at bpos=0-7 context is was byte(3,4) a word, sum of context order(3-5,6,8) isState counts (max 5) and last 2 bit2word 2843 | mxA[3].cxt=((4 * words) & 0xf0) + ord*256 + (w4 & 15); 2844 | 2845 | // mixer 7 2846 | // at bpos=0-7 context is non-repeating 2 bit3word of byte(2,3), was byte(1-3) a word and last 1 bit2word 2847 | mxA[7].cxt=((wtype) & 0x1f8)*4 + ((2 * words) & 0x1c) + (w4 & 3); 2848 | c=c0b; 2849 | // mixer 4 2850 | // at bpos=0 context is bit xxxxxxxx from c0, was byte(1-8) a word or space and bit pos 2851 | // at bpos=1 context is bit 1xxxxxxx from c0, was byte(1-7) a word or space and bit pos 2852 | // at bpos=2 context is bit 11xxxxxx from c0, was byte(1-6) a word or space and bit pos 2853 | // at bpos=3 context is bit 111xxxxx from c0, was byte(1-5) a word or space and bit pos 2854 | // at bpos=4 context is bit 1111xxxx from c0, was byte(1-4) a word or space and bit pos 2855 | // at bpos=5 context is bit 11111xxx from c0, was byte(1-3) a word or space and bit pos 2856 | // at bpos=6 context is bit 111111xx from c0, was byte(1-2) a word or space and bit pos 2857 | // at bpos=7 context is bit 1111111x from c0, was byte(1) a word or space and bit pos 2858 | mxA[4].cxt=bpos*256 + (((( (numbers|words)<< bpos)&255)>> bpos) | (c&255)); 2859 | // mixer 9 - final mixer 2860 | // at bpos=0-7 context is sum of context order(3-5,6,8) isState counts (max 5), bracket or quote state(0,1) and last 1 bit2word 2861 | mxA[9].cxt=(ord*8 + (w4br?1:0)*4 + (w4&3)); 2862 | 2863 | // mixer 5 2864 | // at bpos=0 context is bit xxxxxxxx from c0, first char type state(0,1) xxxx1xxx, 2 bit2word 1111xxxx 2865 | // at bpos=1 context is bit 1xxxxxxx from c0, first char type state(0,1) xxxx1xxx, 1 bit3word x111xxxx, bit pos xxxxx111 2866 | // at bpos=2 context is bit 11xxxxxx from c0, first char type state(0,1) xxxx1xxx, 1 bit2word xx11xxxx, bit pos xxxxx111 2867 | // at bpos=3 context is bit 111xxxxx from c0, first char type state(0,1) xxxx1xxx, was byte(1) a word xxx1xxxx, bit pos xxxxx111 2868 | // at bpos=4 context is bit 1111xxxx from c0, first char type state(0,1) xxxx1xxx, bit pos xxxxx111 2869 | // at bpos=5 context is bit 11111xxx from c0, first char type state(0,1) xxxx1xxx (overflow, ok!) 2870 | // at bpos=6 context is bit 111111xx from c0, first char type state(0,1) xxxx1xxx 2871 | // at bpos=7 context is bit 1111111x from c0, first char type state(0,1) xxxx1xxx 2872 | // at bpos=0-7 sum of context order(3-5,6,8) isState counts (max 5) and is match(0,1) 111 xxxxxxxx 2873 | 2874 | if (bpos) { 2875 | if (bpos==1) { 2876 | c=c + 16*(ttype&7); 2877 | } 2878 | else if (bpos==2) { 2879 | c=c + 16*(w4&3); 2880 | } 2881 | else if (bpos==3) { 2882 | c=c + 16*(words&1); 2883 | } else { 2884 | c=bpos + (c&0xf0); 2885 | } 2886 | if (bpos<5) 2887 | c=bpos + (c&0xf0); 2888 | }else c=16 * (w4&0xf); 2889 | ord=ord-1; 2890 | if (ord<0) 2891 | ord=0; 2892 | if (ismatch) 2893 | ord=ord+1; 2894 | mxA[5].cxt=c + ord*256+ 8*fc1; 2895 | 2896 | // mixer 6 2897 | // at bpos=0-7 context is sum of context words isState counts (max 6), first char type state(0,1), 2 bit2word of byte(3,4) and 1 bit3word of byte(2) 2898 | mxA[6].cxt=(ord2*256 + (w4&0xf0) + ((ttype&0x38) >> 2))*4 + fqcxt; 2899 | 2900 | // mixer 8 2901 | // at bpos 0-2 bit3word, 1 bit3word of current bracket or quote, was byte(1,2,3) a word, first char flag, is a match 2902 | // at bpos 3-7 bit3word from c0, 1 bit3word of current bracket or quote, was byte(1,2,3) a word, first char flag, is a match 2903 | if (bpos>2) 2904 | mxA[8].cxt= wrt_t[c0b&255]*256 +(w4br)*32 + (words&7)*4 + fc1+(ismatch?2:0); 2905 | else 2906 | mxA[8].cxt= (ttype&3)*256 +(w4br)*16 + (words&7)*2 + fc1+(ismatch?128:0); // fixme 2907 | 2908 | x.mxInputs[1].add(mxA[1].p1()); 2909 | x.mxInputs[1].add(mxA[2].p1()); 2910 | x.mxInputs[1].add(mxA[3].p1()); 2911 | x.mxInputs[1].add(mxA[4].p1()); 2912 | x.mxInputs[1].add(mxA[5].p1()); 2913 | x.mxInputs[1].add(mxA[6].p1()); 2914 | x.mxInputs[1].add(mxA[7].p1()); 2915 | x.mxInputs[1].add(mxA[8].p1()); 2916 | return mxA[9].p(); 2917 | } 2918 | 2919 | int rate=6; 2920 | void update1() { 2921 | x.c0+=x.c0+x.y; 2922 | if (x.c0>=256) { 2923 | x.c4=(x.c4<<8)+(x.c0&0xff); 2924 | x.c0=1; 2925 | ++x.blpos; 2926 | // When last byte was predicted good/below error treshold then set new limits to mixer update 2927 | // larger value means less updates and better speed. 2928 | if ((fails&255)==0) { 2929 | for (int i=1;i<9;i++) if (i!=7)mxA[i].elim=max(256,mxA[i].elim+1); 2930 | }else{ 2931 | for (int i=1;i<9;i++) if (i!=7)mxA[i].elim=min(16,mxA[i].elim-1); 2932 | } 2933 | // APM update rate based on input file position 2934 | rate=6 + (x.blpos>14*256*1024) + (x.blpos>28*512*1024); 2935 | } 2936 | x.bpos=(x.bpos+1)&7; 2937 | x.bposshift=7-x.bpos; 2938 | x.c0shift_bpos=(x.c0<<1)^(256>>(x.bposshift)); 2939 | //mxA[0].update(x.y); 2940 | mxA[1].update(x.y); 2941 | mxA[2].update(x.y); 2942 | mxA[3].update(x.y); 2943 | mxA[4].update(x.y); 2944 | mxA[5].update(x.y); 2945 | mxA[6].update(x.y); 2946 | mxA[7].update(x.y); 2947 | mxA[8].update(x.y); 2948 | mxA[9].update(x.y); 2949 | //printf("mixer 0 predictor count %d\n",x.mxInputs[0].ncount); 2950 | x.mxInputs[0].ncount=0; 2951 | //printf("mixer 1 predictor count %d\n",x.mxInputs[1].ncount); 2952 | x.mxInputs[1].ncount=0; 2953 | // This part is from paq8hp12 2954 | if (fails&0x00000080) --failcount; 2955 | fails=fails*2; 2956 | failz=failz*2; 2957 | 2958 | if (x.y) pr=4095-pr; 2959 | if (pr>=e_l[x.bpos]) ++fails, ++failcount; 2960 | if (pr>=848) ++failz; 2961 | 2962 | pr=modelPrediction(x.c0,x.bpos,x.c4); 2963 | AddPrediction(pr); 2964 | 2965 | int pt, pu=(apmA[0].p(pr, x.c0, 3,x.y)+7*pr+4)>>3, pv, pz=failcount+1; 2966 | 2967 | pz+=tri[(fails>>5)&3]; 2968 | pz+=trj[(fails>>3)&3]; 2969 | pz+=trj[(fails>>1)&3]; 2970 | if (fails&1) pz+=8; 2971 | pz=pz/2; 2972 | 2973 | pu=apmA[3].p(pu, ((x.c0*2)^AH1)&0x3ffff, rate,x.y); 2974 | AddPrediction(pu); 2975 | pv=apmA[1].p(pr, ((x.c0*8)^hash(29,failz&2047))&0xffff, rate+1,x.y); 2976 | AddPrediction(pv); 2977 | // If fails use w4 else non-repeating w4 2978 | if (fails&255) 2979 | pv=apmA[4].p(pv, hash(x.c0,w4 & 0xfffc,(wtype & 0x1ff))&0x1ffff, rate,x.y); 2980 | else 2981 | pv=apmA[4].p(pv, hash(x.c0,(w4r & 0xfffc)+0x10000,(wtype & 0x1ff))&0x1ffff, rate,x.y); 2982 | AddPrediction(pv); 2983 | pt=apmA[2].p(pr, ( (x.c0*32)^AH2)&0xffff, rate,x.y); 2984 | AddPrediction(pt); 2985 | pz=apmA[5].p(pu, ((x.c0*4)^hash(min(9,pz),x5&0x80ff))&0x1ffff, rate,x.y); 2986 | AddPrediction(pz); 2987 | if (fails&255) pr=(pt*6+pu +pv*11+pz*14 +31)>>5; 2988 | else pr=(pt*4+pu*5+pv*12+pz*11 +31)>>5; 2989 | AddPrediction(pr); 2990 | } 2991 | 2992 | 2993 | class Predictor { 2994 | //int pr; 2995 | public: 2996 | Predictor(); 2997 | int p() const {return pr;} 2998 | void update(); 2999 | }; 3000 | 3001 | Predictor::Predictor() { 3002 | 3003 | // Precalculate tabeles 3004 | int o=2; 3005 | for (int i=0; i<1024; ++i) 3006 | dt[i]=4096/(o),o++; 3007 | dt[1023]=1; 3008 | 3009 | // Stretch table 3010 | for (int i=0; i<=4095; i++) { 3011 | strt[i]=stretchc(i); 3012 | } 3013 | 3014 | // Squash table 3015 | for (int i=-2047; i<=2047; i++) { 3016 | sqt[i+2047]=squashc(i); 3017 | } 3018 | 3019 | InitIlog(); 3020 | x.Init(); 3021 | 3022 | // Match model 3023 | mhashtablemask=0x200000*1-1; 3024 | alloc1(mhashtable,0x200000*1+32,mhptr,32); 3025 | PredictorInit(); 3026 | } 3027 | 3028 | void Predictor::update() { 3029 | 3030 | update1(); 3031 | ResetPredictions(); 3032 | } 3033 | 3034 | } 3035 | 3036 | FXCM::FXCM() { 3037 | predictor_.reset(new fxcmv1::Predictor()); 3038 | } 3039 | 3040 | const std::valarray& FXCM::Predict() { 3041 | return fxcmv1::model_predictions; 3042 | } 3043 | 3044 | unsigned int FXCM::NumOutputs() { 3045 | return fxcmv1::model_predictions.size(); 3046 | } 3047 | 3048 | void FXCM::Perceive(int bit) { 3049 | fxcmv1::x.y = bit; 3050 | predictor_->update(); 3051 | } 3052 | 3053 | -------------------------------------------------------------------------------- /for cmix/fxcmv1.h: -------------------------------------------------------------------------------- 1 | #ifndef FXCM_H 2 | #define FXCM_H 3 | 4 | #include "model.h" 5 | #include 6 | #include 7 | 8 | namespace fxcmv1 { 9 | class Predictor; 10 | } 11 | 12 | class FXCM : public Model { 13 | public: 14 | FXCM(); 15 | const std::valarray& Predict(); 16 | unsigned int NumOutputs(); 17 | void Perceive(int bit); 18 | void ByteUpdate() {}; 19 | 20 | private: 21 | std::unique_ptr predictor_; 22 | }; 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /for cmix/readme.txt: -------------------------------------------------------------------------------- 1 | This is model for cmix-hp for replacing paq8hp. 2 | 3 | Notes: 4 | In predictor.c interval6 map should point to wrt_t and interval4 wrt_w in fxcmv1.cpp. 5 | 6 | Model expexts input to be processed like cmix -c and some chars swaped. 7 | By default #TEXTMODE is disabled for this. 8 | If no dictionary is used then #TEXTMODE should be enabled. This is compile time option. 9 | 10 | If this model is used then paq8hp should be disabled. 11 | Not tested under Linux so may need adjustments. -------------------------------------------------------------------------------- /tools/states.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #define NDEBUG 4 | #include 5 | #include 6 | 7 | 8 | class StateTable { 9 | int mdc; // maximum discount 10 | enum {B=5, N=64}; // sizes of b, t 11 | int b[6]; // x -> max y, y -> max x 12 | unsigned char ns[1024]; // state*4 -> next state if 0, if 1, n0, n1 13 | unsigned char t[N][N][2]={{{0}}}; 14 | int num_states(int x, int y); // compute t[x][y][1] 15 | void discount(int& x); // set new value of x after 1 or y after 0 16 | void next_state(int& x, int& y, int b); // new (x,y) after bit b 17 | void generate(); // compute t[x][y][1] 18 | int pre(int state) { // initial probability of 1 * 2^23 19 | assert(state>=0 && state<256); 20 | return ((next(state,3)*2+1)<<14)/(next(state,2)+next(state,3)+1); 21 | } 22 | public: 23 | int next(int state, int sel) {return ns[state*4+sel];} 24 | StateTable(); 25 | Init(int s0,int s1,int s2,int s3,int s4,int s5,int s6); 26 | } ; 27 | 28 | 29 | int StateTable::num_states(int x, int y) { 30 | if (x=N || y>=N || y>=B || x>=b[y]) return 0; 32 | return 1+(y>0 && x+y2){ 39 | for (int i=1;i=i; 40 | x=y; 41 | } 42 | } 43 | 44 | // compute next x,y (0 to N) given input b (0 or 1) 45 | void StateTable::next_state(int& x, int& y, int b) { 46 | if (x next if 0, next if 1, x, y 68 | void StateTable::generate() { 69 | memset(ns, 0, sizeof(ns)); 70 | memset(t, 0, sizeof(t)); 71 | // Assign states 72 | int state=0; 73 | for (int i=0; i<256; ++i) { 74 | for (int y=0; y<=i; ++y) { 75 | int x=i-y; 76 | int n=num_states(x, y); 77 | if (n) { 78 | t[x][y][0]=state; 79 | t[x][y][1]=n; 80 | state+=n; 81 | } 82 | } 83 | } 84 | 85 | // Print/generate next state table 86 | state=0; 87 | for (int i=0; i1); 97 | ns[state*4+2]=x; 98 | ns[state*4+3]=y; 99 | 100 | // uncomment to print table above 101 | printf("{%3d,%3d,%2d,%2d},", ns[state*4], ns[state*4+1], 102 | ns[state*4+2], ns[state*4+3]); 103 | if (state%4==3) printf(" // %d-%d\n", state-3, state); 104 | if (state>0xff || t[x][y][1]==0 || t[x0][y0][1]==0 || t[x1][y1][1]==0) return; 105 | assert(state>=0 && state<256); 106 | assert(t[x][y][1]>0); 107 | assert(t[x][y][0]<=state); 108 | assert(t[x][y][0]+t[x][y][1]>state); 109 | assert(t[x][y][1]<=6); 110 | assert(t[x0][y0][1]>0); 111 | assert(t[x1][y1][1]>0); 112 | assert(ns0-t[x0][y0][0]=0); 114 | assert(ns1-t[x1][y1][0]=0); 116 | ++state; 117 | } 118 | } 119 | } 120 | } 121 | 122 | StateTable::StateTable() { 123 | } 124 | 125 | StateTable::Init(int s0,int s1,int s2,int s3,int s4,int s5,int s6) { 126 | b[0]=s0;b[1]=s1;b[2]=s2;b[3]=s3;b[4]=s4;b[5]=s5;mdc=s6; 127 | printf("\n Generating with state parameters p1 %d,p2 %d,p3 %d,p4 %d,p5 %d,p6 %d,p7 %d\n",s0,s1,s2,s3,s4,s5,s6); 128 | printf("Statetable:\n"); 129 | generate(); 130 | printf("\n"); 131 | } 132 | StateTable stable; 133 | 134 | 135 | int main(int argc, char** argv) { 136 | if (argc!=8) { 137 | printf("Usage:\nstates p1 p2 p3 p4 p5 p6 p7\n"); 138 | exit(1); 139 | } 140 | stable.Init(atoi(argv[1]) ,atoi(argv[2]) ,atoi(argv[3]) ,atoi(argv[4]) ,atoi(argv[5]), atoi(argv[6]),atoi(argv[7]) ); 141 | 142 | return 0; 143 | } 144 | --------------------------------------------------------------------------------