├── COPYING ├── LICENSE ├── README ├── README.md ├── examples ├── data │ ├── elec-test.cat │ ├── elec-test.txt.tok │ ├── elec-train.cat │ ├── elec-train.txt.tok │ ├── elec_cat.dic │ ├── imdb-test.cat │ ├── imdb-train.cat │ ├── imdb_cat.dic │ ├── imdb_registered2.txt │ ├── minstop.txt │ ├── rcv1-lvl2.catdic │ ├── rcv1_stopword.txt │ ├── s-cat.dic │ ├── s-dim10.wvbin │ ├── s-dim10.wvtxt │ ├── s-dp-dv.cat │ ├── s-dp-dv.txt.tok │ ├── s-dp-td.1of2.cat │ ├── s-dp-td.1of2.txt.tok │ ├── s-dp-td.2of2.cat │ ├── s-dp-td.2of2.txt.tok │ ├── s-dp-test.cat │ ├── s-dp-test.txt │ ├── s-dp-test.txt.tok │ ├── s-dp-train.cat │ ├── s-dp-train.txt │ ├── s-dp.catdic │ ├── s-multilab-test.cat │ ├── s-multilab-test.txt.tok │ ├── s-multilab-train.cat │ ├── s-multilab-train.txt.tok │ ├── s-multilab.catdic │ ├── s-test.cat │ ├── s-test.txt.tok │ ├── s-train.cat │ └── s-train.txt.tok ├── dpcnn-functions.sh ├── dpcnn-tokenize.sh ├── dpcnn-with-1unsemb.sh ├── dpcnn-with-4unsemb.sh ├── dpcnn-without-unsemb.sh ├── extract_text-imdb.pl ├── ld-extract.pl ├── ld-functions.sh ├── lst │ ├── imdb-test-neg.lst │ ├── imdb-test-pos.lst │ ├── imdb-train-neg.lst │ └── imdb-train-pos.lst ├── lstm-2unsemb-imdb-elec.sh ├── lstm-5unsemb-imdb-elec.sh ├── lstm-imdb-elec.sh ├── merge_dic.pl ├── other-sh │ ├── lstm-2unsemb-rcv1.sh │ ├── lstm-5unsemb-rcv1.sh │ ├── lstm-rcv1-20ng.sh │ ├── prep_elec.sh │ ├── prep_imdb.sh │ ├── shcnn-3unsemb-rcv1.sh │ ├── shcnn-5unsemb-rcv1.sh │ ├── shcnn-bow-all-rcv1.sh │ ├── shcnn-bow-rcv1.sh │ ├── shcnn-parsup-rcv1.sh │ ├── shcnn-seq2-3k-imdb.sh │ ├── shcnn-seq2-all-elec.sh │ ├── shcnn-unsup-rcv1.sh │ └── shcnn-unsup3-rcv1.sh ├── sample-multilab.sh ├── sample-save-predict.sh ├── sample-wordvec.sh ├── sample.sh ├── sh-common.sh ├── shcnn-3unsemb-imdb-elec.sh ├── shcnn-5unsemb-imdb-elec.sh ├── shcnn-parsup-imdb-elec.sh ├── shcnn-seq-imdb-elec.sh ├── shcnn-seq2-bown-imdb-elec.sh ├── shcnn-seq2-imdb-elec.sh ├── shcnn-unsup-imdb-elec.sh ├── shcnn-unsup3-imdb-elec.sh └── to_tokens.pl ├── imdb-data.tar.gz ├── makefile ├── makefile7.5 └── src ├── com ├── AzDic.hpp ├── AzDmat.cpp ├── AzDmat.hpp ├── AzException.hpp ├── AzHelp.hpp ├── AzMatVar.hpp ├── AzMemTempl.hpp ├── AzOut.hpp ├── AzParam.cpp ├── AzParam.hpp ├── AzPrint.hpp ├── AzRandGen.hpp ├── AzReadOnlyMatrix.hpp ├── AzSmat.cpp ├── AzSmat.hpp ├── AzStrPool.cpp ├── AzStrPool.hpp ├── AzTextMat.cpp ├── AzTextMat.hpp ├── AzTools.cpp ├── AzTools.hpp ├── AzUtil.cpp └── AzUtil.hpp ├── data ├── AzPrepText.cpp ├── AzPrepText.hpp ├── AzTools_text.cpp ├── AzTools_text.hpp └── driv_PrepText.cpp └── nnet ├── Az2D.hpp ├── AzCuda.cuh ├── AzCuda_Pmat.cu ├── AzCuda_Pmat.cuh ├── AzCuda_PmatApp.cu ├── AzCuda_PmatApp.cuh ├── AzCuda_PmatSpa.cu ├── AzCuda_PmatSpa.cuh ├── AzMultiConn.cpp ├── AzMultiConn.hpp ├── AzP.h ├── AzPmat.cpp ├── AzPmat.hpp ├── AzPmatApp.cpp ├── AzPmatApp.hpp ├── AzPmatApp_gpu.cuh ├── AzPmatSpa.cpp ├── AzPmatSpa.hpp ├── AzPmatSpa_gpu.cu ├── AzPmatSpa_gpu.cuh ├── AzPmat_gpu.cu ├── AzPmat_gpu.cuh ├── AzPmem.cuh ├── AzStepszSch.hpp ├── AzpActivDflt.hpp ├── AzpActiv_.hpp ├── AzpCompoSetDflt.hpp ├── AzpCompoSet_.hpp ├── AzpCompo_.hpp ├── AzpDataSeq.hpp ├── AzpDataSetDflt.hpp ├── AzpData_.hpp ├── AzpData_img.hpp ├── AzpData_imgbin.hpp ├── AzpData_sparse.hpp ├── AzpData_sparse_multi.hpp ├── AzpData_tmpl_.hpp ├── AzpDropoutDflt.hpp ├── AzpDropout_.hpp ├── AzpEv.cpp ├── AzpEv.hpp ├── AzpLm.hpp ├── AzpLmAdaD.hpp ├── AzpLmParam.hpp ├── AzpLmRmsp.hpp ├── AzpLmSgd.cpp ├── AzpLmSgd.hpp ├── AzpLossDflt.cpp ├── AzpLossDflt.hpp ├── AzpLoss_.hpp ├── AzpMain_reNet.cpp ├── AzpMain_reNet.hpp ├── AzpPatchDflt.cpp ├── AzpPatchDflt.hpp ├── AzpPatchDflt_var.hpp ├── AzpPatch_.hpp ├── AzpPatch_var_.hpp ├── AzpPoolingDflt.hpp ├── AzpPoolingDflt_var.hpp ├── AzpPooling_.hpp ├── AzpPooling_var_.hpp ├── AzpReLayer.cpp ├── AzpReLayer.hpp ├── AzpReNet.cpp ├── AzpReNet.hpp ├── AzpResNormDflt.hpp ├── AzpResNorm_.hpp ├── AzpTimer_CNN.hpp ├── AzpWeightDflt.hpp ├── AzpWeight_.hpp ├── AzpWeight_sgdlike.hpp ├── AzxD.hpp └── driv_reNet.cpp /README.md: -------------------------------------------------------------------------------- 1 | ## ConText v4.00: C++ program for neural networks for text categorization 2 | ConText v4.00 provides a C++ implementation of neural networks for text categorization described in: 3 | * [Deep pyramid convolutional neural networks for text categorization](http://aclweb.org/anthology/P/P17/P17-1052.pdf). Rie Johnson and Tong Zhang. ACL 2017. 4 | * [Effective use of word order for text categorization with convolutional neural networks](https://aclweb.org/anthology/N/N15/N15-1011.pdf). Rie Johnson and Tong Zhang. NAACL HLT 2015. 5 | * [Semi-supervised convolutional neural networks for text categorization via region embedding](https://papers.nips.cc/paper/5849-semi-supervised-convolutional-neural-networks-for-text-categorization-via-region-embedding). Rie Johnson and Tong Zhang. NIPS 2015. 6 | * [Supervised and semi-supervised text categorization using LSTM for region embeddings](http://proceedings.mlr.press/v48/johnson16.pdf). Rie Johnson and Tong Zhang. ICML 2016. 7 | 8 | ConText v4.00 is available at http://riejohnson.com/cnn_download.html. 9 | 10 | **_System Requirements_**: This software runs only on a CUDA-capable GPU such as Tesla K20. That is, your system **must have a GPU** and an appropriate version of CUDA installed. The provided `makefile` and example shell scripts are for Unix-like systems. Testing was done on Linux. In principle, the C++ code should compile and run also in other systems (e.g., Windows), but no guarantee. See [`README`](README) for more details. 11 | 12 | **_Download & Documentation_**: See http://riejohnson.com/cnn_download.html#download. 13 | 14 | **_Getting Started_** 15 | 1. Download the code and extract the files, and read [`README`](README) (not `README.md`). 16 | 2. Go to the top directory and build executables by entering `make`, after customizing `makefile` as needed. 17 | (If you downloaded from GitHub, `make` also decompresses sample text files that exceed GitHub file size limit 18 | and does `chmod +x` on shell scripts.) 19 | 3. To confirm installation, go to `examples/` and enter `./sample.sh`. 20 | (See [`README`](README) for installation trouble shooting.) 21 | 4. Read Section 1 (Overview) of [User Guide](http://riejohnson.com/software/conText-v4-ug.pdf) to get an idea. 22 | 5. Try some shell scripts at `examples/`. There is a table of the scripts in Section 1.6 of 23 | [User Guide](http://riejohnson.com/software/conText-v4-ug.pdf). 24 | 25 | **_Data Source_**: The data files were derived from [Large Move Review Dataset (IMDB)](http://ai.stanford.edu/~amaas/data/sentiment/) 26 | [MDPHN11] and [Amazon reviews](http://snap.stanford.edu/data/web-Amazon.html) [ML13]. 27 | 28 | **_Licence_**: This program is free software issued under the [GNU General Public License V3](http://www.gnu.org/copyleft/gpl.html). 29 | 30 | **_References_** 31 | [MDPHN11] Andrew L. Maas, Raymond E. Daly, Peter T. Pham, Dan Huang, Andrew Y. Ng, and Christopher Potts. Learning word vectors for sentiment analysis. ACL 2011. 32 | [ML13] Julian McAuley and Jure Leskovec. Hidden factors and hidden topics: understanding rating dimensions with review text. RecSys 2013. 33 | 34 | **_Note_**: This GitHub repository provides a snapshot of research code, which is constantly changing elsewhere for research purposes. For this reason, it is very likely that pull requests will be declined. 35 | -------------------------------------------------------------------------------- /examples/data/elec_cat.dic: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | -------------------------------------------------------------------------------- /examples/data/imdb_cat.dic: -------------------------------------------------------------------------------- 1 | neg 2 | pos 3 | -------------------------------------------------------------------------------- /examples/data/imdb_registered2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/riejohnson/ConText/4f51d8db5bffa8278237353f3a6bfa2ee9785071/examples/data/imdb_registered2.txt -------------------------------------------------------------------------------- /examples/data/minstop.txt: -------------------------------------------------------------------------------- 1 | $-$ 2 | ~ 3 | @ 4 | # 5 | ^ 6 | & 7 | * 8 | ( 9 | ) 10 | - 11 | _ 12 | + 13 | = 14 | { 15 | } 16 | [ 17 | ] 18 | \ 19 | | 20 | ; 21 | : 22 | " 23 | ' 24 | , 25 | . 26 | < 27 | > 28 | / 29 | about 30 | across 31 | against 32 | along 33 | around 34 | at 35 | behind 36 | beside 37 | by 38 | despite 39 | down 40 | during 41 | for 42 | from 43 | in 44 | inside 45 | into 46 | near 47 | of 48 | off 49 | on 50 | onto 51 | over 52 | through 53 | to 54 | toward 55 | with 56 | within 57 | without 58 | i 59 | you 60 | he 61 | she 62 | me 63 | him 64 | my 65 | mine 66 | her 67 | hers 68 | his 69 | myself 70 | himself 71 | it 72 | we 73 | they 74 | us 75 | them 76 | our 77 | ours 78 | their 79 | theirs 80 | itself 81 | ourselves 82 | themselves 83 | the 84 | this 85 | that 86 | a 87 | an 88 | and 89 | but 90 | after 91 | when 92 | as 93 | because 94 | if 95 | what 96 | where 97 | which 98 | how 99 | than 100 | or 101 | before 102 | since 103 | while 104 | although 105 | who 106 | whose 107 | can 108 | may 109 | will 110 | shall 111 | could 112 | might 113 | would 114 | should 115 | must 116 | be 117 | do 118 | have 119 | am 120 | is 121 | are 122 | was 123 | were 124 | been 125 | did 126 | done 127 | has 128 | had 129 | 's 130 | 'd 131 | 'll 132 | 've 133 | 're 134 | 'm -------------------------------------------------------------------------------- /examples/data/rcv1-lvl2.catdic: -------------------------------------------------------------------------------- 1 | C11 2 | C12 3 | C13 4 | C14 5 | C15 6 | C16 7 | C17 8 | C18 9 | C21 10 | C22 11 | C23 12 | C24 13 | C31 14 | C32 15 | C33 16 | C34 17 | C41 18 | C42 19 | E11 20 | E12 21 | E13 22 | E14 23 | E21 24 | E31 25 | E41 26 | E51 27 | E61 28 | E71 29 | G15 30 | GCRIM 31 | GDEF 32 | GDIP 33 | GDIS 34 | GENT 35 | GENV 36 | GFAS 37 | GHEA 38 | GJOB 39 | GMIL 40 | GOBIT 41 | GODD 42 | GPOL 43 | GPRO 44 | GREL 45 | GSCI 46 | GSPO 47 | GTOUR 48 | GVIO 49 | GVOTE 50 | GWEA 51 | GWELF 52 | M11 53 | M12 54 | M13 55 | M14 -------------------------------------------------------------------------------- /examples/data/s-cat.dic: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | -------------------------------------------------------------------------------- /examples/data/s-dim10.wvbin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/riejohnson/ConText/4f51d8db5bffa8278237353f3a6bfa2ee9785071/examples/data/s-dim10.wvbin -------------------------------------------------------------------------------- /examples/data/s-dp.catdic: -------------------------------------------------------------------------------- 1 | neg 2 | pos 3 | -------------------------------------------------------------------------------- /examples/data/s-multilab-test.cat: -------------------------------------------------------------------------------- 1 | 2 | 2 3 | 2 4 | 2 5 | 2 6 | 1 7 | 1 8 | 2 9 | 2 10 | 2 11 | 2 12 | 2 13 | 2 14 | 1 15 | 2 16 | 1 17 | 2 18 | 2 19 | 2 20 | 1 21 | 2 22 | 1 23 | 3|2 24 | 1 25 | 1 26 | 2 27 | 1 28 | 2 29 | 2 30 | 1 31 | 2 32 | 2 33 | 2 34 | 2 35 | 2 36 | 2 37 | 2 38 | 2 39 | 2 40 | 2 41 | 2 42 | 2 43 | 2 44 | 2 45 | 2 46 | 2 47 | 2 48 | 2 49 | 2 50 | 2 51 | 1 52 | 2 53 | 1 54 | 2 55 | 2 56 | 1 57 | 1 58 | 2 59 | 2 60 | 2 61 | 2 62 | 1 63 | 2 64 | 2 65 | 2 66 | 2 67 | 2 68 | 2 69 | 2 70 | 2 71 | 2 72 | 2 73 | 2 74 | 2 75 | 2 76 | 2 77 | 2 78 | 2 79 | 1 80 | 2 81 | 1 82 | 2 83 | 2 84 | 2 85 | 1 86 | 1 87 | 2 88 | 2 89 | 2 90 | 2 91 | 2 92 | 2 93 | 2 94 | 2 95 | 2 96 | 2 97 | 1 98 | 2 99 | 2 100 | 2 101 | 2 102 | 2 103 | 2 104 | 2 105 | 2 106 | 2 107 | 2 108 | 2 109 | 2 110 | 2 111 | 2 112 | 2 113 | 2 114 | 2 115 | 2 116 | 1 117 | 1 118 | 2 119 | 2 120 | 2 121 | 2 122 | 2 123 | 2 124 | 2 125 | 2 126 | 2 127 | 2 128 | 1 129 | 2 130 | 1 131 | 1 132 | 2 133 | 1 134 | 2 135 | 2 136 | 1 137 | 2 138 | 2 139 | 1 140 | 1 141 | 2 142 | 2 143 | 2 144 | 2 145 | 2 146 | 2 147 | 1 148 | 2 149 | 2 150 | 2 151 | 2 152 | 2 153 | 2 154 | 2 155 | 2 156 | 2 157 | 2 158 | 2 159 | 2 160 | 2 161 | 2 162 | 2 163 | 1 164 | 2 165 | 2 166 | 1 167 | 2 168 | 1 169 | 2 170 | 2 171 | 1 172 | 1 173 | 2 174 | 1 175 | 2 176 | 2 177 | 2 178 | 2 179 | 1 180 | 2 181 | 2 182 | 2 183 | 2 184 | 1 185 | 2 186 | 2 187 | 2 188 | 1 189 | 1 190 | 1 191 | 1 192 | 2 193 | 2 194 | 2 195 | 2 196 | 1 197 | 2 198 | 1 199 | 2 200 | 1 201 | 1 202 | 2 203 | 1 204 | 2 205 | 2 206 | 2 207 | 2 208 | 2 209 | 2 210 | 2 211 | 1 212 | 2 213 | 2 214 | 2 215 | 2 216 | 2 217 | 2 218 | 2 219 | 2 220 | 2 221 | 2 222 | 2 223 | 2 224 | 1 225 | 2 226 | 2 227 | 2 228 | 2 229 | 2 230 | 2 231 | 2 232 | 2 233 | 1 234 | 2 235 | 2 236 | 2 237 | 2 238 | 2 239 | 2 240 | 2 241 | 2 242 | 2 243 | 2 244 | 2 245 | 2 246 | 2 247 | 2 248 | 2 249 | 2 250 | 2 251 | 1 252 | 1 253 | 1 254 | 1 255 | 1 256 | 1 257 | 1 258 | 1 259 | 1 260 | 1 261 | 1 262 | 1 263 | 1 264 | 1 265 | 1 266 | 1 267 | 1 268 | 1 269 | 1 270 | 1 271 | 1 272 | 1 273 | 1 274 | 1 275 | 1 276 | 1 277 | 1 278 | 1 279 | 1 280 | 1 281 | 1 282 | 1 283 | 1 284 | 1 285 | 1 286 | 1 287 | 1 288 | 1 289 | 1 290 | 1 291 | 1 292 | 1 293 | 1 294 | 1 295 | 1 296 | 1 297 | 1 298 | 1 299 | 1 300 | 1 301 | 1 302 | 1 303 | 1 304 | 1 305 | 1 306 | 1 307 | 1 308 | 1 309 | 1 310 | 1 311 | 1 312 | 1 313 | 1 314 | 1 315 | 1 316 | 1 317 | 1 318 | 1 319 | 1 320 | 1 321 | 1 322 | 1 323 | 1 324 | 1 325 | 1 326 | 1 327 | 1 328 | 1 329 | 1 330 | 1 331 | 1 332 | 1 333 | 1 334 | 1 335 | 1 336 | 1 337 | 1 338 | 1 339 | 1 340 | 1 341 | 1 342 | 1 343 | 1 344 | 1 345 | 1 346 | 1 347 | 1 348 | 1 349 | 1 350 | 1 351 | 1 352 | 1 353 | 1 354 | 1 355 | 1 356 | 1 357 | 1 358 | 1 359 | 1 360 | 1 361 | 1 362 | 1 363 | 1 364 | 1 365 | 1 366 | 1 367 | 1 368 | 1 369 | 1 370 | 1 371 | 1 372 | 1 373 | 1 374 | 1 375 | 1 376 | 1 377 | 1 378 | 1 379 | 1 380 | 1 381 | 1 382 | 1 383 | 1 384 | 1 385 | 1 386 | 1 387 | 1 388 | 1 389 | 1 390 | 1 391 | 1 392 | 1 393 | 1 394 | 1 395 | 1 396 | 1 397 | 1 398 | 1 399 | 1 400 | 1 401 | -------------------------------------------------------------------------------- /examples/data/s-multilab.catdic: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | 3 4 | -------------------------------------------------------------------------------- /examples/data/s-test.cat: -------------------------------------------------------------------------------- 1 | 2 2 | 2 3 | 2 4 | 2 5 | 2 6 | 1 7 | 1 8 | 2 9 | 2 10 | 2 11 | 2 12 | 2 13 | 2 14 | 1 15 | 2 16 | 1 17 | 2 18 | 2 19 | 2 20 | 1 21 | 2 22 | 1 23 | 2 24 | 1 25 | 1 26 | 2 27 | 1 28 | 2 29 | 2 30 | 1 31 | 2 32 | 2 33 | 2 34 | 2 35 | 2 36 | 2 37 | 2 38 | 2 39 | 2 40 | 2 41 | 2 42 | 2 43 | 2 44 | 2 45 | 2 46 | 2 47 | 2 48 | 2 49 | 2 50 | 2 51 | 1 52 | 2 53 | 1 54 | 2 55 | 2 56 | 1 57 | 1 58 | 2 59 | 2 60 | 2 61 | 2 62 | 1 63 | 2 64 | 2 65 | 2 66 | 2 67 | 2 68 | 2 69 | 2 70 | 2 71 | 2 72 | 2 73 | 2 74 | 2 75 | 2 76 | 2 77 | 2 78 | 2 79 | 1 80 | 2 81 | 1 82 | 2 83 | 2 84 | 2 85 | 1 86 | 1 87 | 2 88 | 2 89 | 2 90 | 2 91 | 2 92 | 2 93 | 2 94 | 2 95 | 2 96 | 2 97 | 1 98 | 2 99 | 2 100 | 2 101 | 2 102 | 2 103 | 2 104 | 2 105 | 2 106 | 2 107 | 2 108 | 2 109 | 2 110 | 2 111 | 2 112 | 2 113 | 2 114 | 2 115 | 2 116 | 1 117 | 1 118 | 2 119 | 2 120 | 2 121 | 2 122 | 2 123 | 2 124 | 2 125 | 2 126 | 2 127 | 2 128 | 1 129 | 2 130 | 1 131 | 1 132 | 2 133 | 1 134 | 2 135 | 2 136 | 1 137 | 2 138 | 2 139 | 1 140 | 1 141 | 2 142 | 2 143 | 2 144 | 2 145 | 2 146 | 2 147 | 1 148 | 2 149 | 2 150 | 2 151 | 2 152 | 2 153 | 2 154 | 2 155 | 2 156 | 2 157 | 2 158 | 2 159 | 2 160 | 2 161 | 2 162 | 2 163 | 1 164 | 2 165 | 2 166 | 1 167 | 2 168 | 1 169 | 2 170 | 2 171 | 1 172 | 1 173 | 2 174 | 1 175 | 2 176 | 2 177 | 2 178 | 2 179 | 1 180 | 2 181 | 2 182 | 2 183 | 2 184 | 1 185 | 2 186 | 2 187 | 2 188 | 1 189 | 1 190 | 1 191 | 1 192 | 2 193 | 2 194 | 2 195 | 2 196 | 1 197 | 2 198 | 1 199 | 2 200 | 1 201 | 1 202 | 2 203 | 1 204 | 2 205 | 2 206 | 2 207 | 2 208 | 2 209 | 2 210 | 2 211 | 1 212 | 2 213 | 2 214 | 2 215 | 2 216 | 2 217 | 2 218 | 2 219 | 2 220 | 2 221 | 2 222 | 2 223 | 2 224 | 1 225 | 2 226 | 2 227 | 2 228 | 2 229 | 2 230 | 2 231 | 2 232 | 2 233 | 1 234 | 2 235 | 2 236 | 2 237 | 2 238 | 2 239 | 2 240 | 2 241 | 2 242 | 2 243 | 2 244 | 2 245 | 2 246 | 2 247 | 2 248 | 2 249 | 2 250 | 2 251 | 1 252 | 1 253 | 1 254 | 1 255 | 1 256 | 1 257 | 1 258 | 1 259 | 1 260 | 1 261 | 1 262 | 1 263 | 1 264 | 1 265 | 1 266 | 1 267 | 1 268 | 1 269 | 1 270 | 1 271 | 1 272 | 1 273 | 1 274 | 1 275 | 1 276 | 1 277 | 1 278 | 1 279 | 1 280 | 1 281 | 1 282 | 1 283 | 1 284 | 1 285 | 1 286 | 1 287 | 1 288 | 1 289 | 1 290 | 1 291 | 1 292 | 1 293 | 1 294 | 1 295 | 1 296 | 1 297 | 1 298 | 1 299 | 1 300 | 1 301 | 1 302 | 1 303 | 1 304 | 1 305 | 1 306 | 1 307 | 1 308 | 1 309 | 1 310 | 1 311 | 1 312 | 1 313 | 1 314 | 1 315 | 1 316 | 1 317 | 1 318 | 1 319 | 1 320 | 1 321 | 1 322 | 1 323 | 1 324 | 1 325 | 1 326 | 1 327 | 1 328 | 1 329 | 1 330 | 1 331 | 1 332 | 1 333 | 1 334 | 1 335 | 1 336 | 1 337 | 1 338 | 1 339 | 1 340 | 1 341 | 1 342 | 1 343 | 1 344 | 1 345 | 1 346 | 1 347 | 1 348 | 1 349 | 1 350 | 1 351 | 1 352 | 1 353 | 1 354 | 1 355 | 1 356 | 1 357 | 1 358 | 1 359 | 1 360 | 1 361 | 1 362 | 1 363 | 1 364 | 1 365 | 1 366 | 1 367 | 1 368 | 1 369 | 1 370 | 1 371 | 1 372 | 1 373 | 1 374 | 1 375 | 1 376 | 1 377 | 1 378 | 1 379 | 1 380 | 1 381 | 1 382 | 1 383 | 1 384 | 1 385 | 1 386 | 1 387 | 1 388 | 1 389 | 1 390 | 1 391 | 1 392 | 1 393 | 1 394 | 1 395 | 1 396 | 1 397 | 1 398 | 1 399 | 1 400 | 1 401 | -------------------------------------------------------------------------------- /examples/dpcnn-tokenize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #**************************************** 3 | # ld_split_tokenize (a functino in ld-functions.sh) does the following: 4 | # - Extract text and labels from csv files of Yelp.p etc. 5 | # - Split training data into a training portion and a validation portion. 6 | # - Tokenize text. 7 | # - Generate a label dictionary. 8 | # 9 | # * Naming conventions for output files ( see data/s-dp.* for example) 10 | # - training data: ${nm}-td.${no}of${bb}.{txt.tok|cat} where bb is the number of batches, no=1,2,...,bb 11 | # - validation data: ${nm}-dv.{txt.tok|cat} 12 | # - test data: ${nm}-test.{txt.tok|cat} 13 | # - label dictionary:${nm}.catdic 14 | #**************************************** 15 | 16 | #-----------------# 17 | source sh-common.sh # Constants 18 | source ld-functions.sh # Functions for large data. 19 | #-----------------# 20 | ddir=csv-data # <= Change this to where the csv data directories are 21 | max_jobs=5 # <= Change this to the desired number of background processes invoked at once. 22 | 23 | #--- parameters for ld_split_tokenize 24 | dev=10 # Generate a development set of 10K documents from the training set. 25 | reg=data/imdb_registered2.txt 26 | odir=gendata; if [ ! -e $odir ]; then mkdir $odir; fi # output directory. 27 | 28 | # nm: output pathnames will be ${tmpdir}/${onm}.* 29 | # num: number of training documents divided by 1000 30 | # bb: number of batches to be generated. 31 | # do_extract: extract text and labels from csv files. 32 | 33 | #--- uncomment one of the following 8 lines 34 | # orgdir=${ddir}/yelp_review_polarity_csv; nm=yelppol; num=560; bb=5 35 | # orgdir=${ddir}/yelp_review_full_csv; nm=yelpfull; num=650; bb=5 36 | # orgdir=${ddir}/dbpedia_csv; nm=dbpedia; num=560; bb=5; do_dbpedia=1 37 | # orgdir=${ddir}/amazon_review_polarity_csv; nm=amapol; num=3600; bb=10 38 | # orgdir=${ddir}/amazon_review_full_csv; nm=amafull; num=3000; bb=10 39 | # orgdir=${ddir}/sogou_news_csv; nm=sog10; num=450; bb=10 40 | # orgdir=${ddir}/yahoo_answers_csv; nm=yah10; num=1400; bb=10 41 | # orgdir=${ddir}/ag_news_csv; nm=ag; num=120; bb=1 42 | 43 | #--- - Extract text and labels from csv files and generate a label dictionary. 44 | #--- - Split training data and the training portion and validation (development) portion. 45 | #--- - Tokenize text. 46 | ld_split_tokenize 47 | if [ $? != 0 ]; then echo $shnm: ld_split_tokenize failed.; exit 1; fi 48 | 49 | -------------------------------------------------------------------------------- /examples/dpcnn-with-1unsemb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #--------------------- 3 | # DPCNN with unsupervised embedding. 4 | # This script uses miniscule toy data and train DPCNN with one unsupervised embedding. 5 | # 6 | # NOTE1: For light-weight demonstration, toy data is used here, and note that 7 | # DPCNN is NOT suitable for miniscule data like this one. 8 | # NOTE2: Parameters used in [JZ17] on real-world data such as Yelp.p are 9 | # in dpcnn-with-unsemb.sh 10 | # 11 | # Step 1. Train an unsupervised embedding function. 12 | # Step 2. Train DPCNN using the unsupervised embedding trained above to produce additional input. 13 | # Step 3. Apply a model to the test data and write prediction values to a file. 14 | #--------------------- 15 | source sh-common.sh 16 | source dpcnn-functions.sh # Functions for DPCNN 17 | 18 | gpu=-1 # <= Change this to, e.g., "gpu=0" to use a specific GPU. 19 | max_jobs=5 # <= Data preparation (dpcnn_gen_regions*) creates up to this number of background processes. 20 | dont_reuse=1 # <= 1: Don't reuse old files and create new files. 21 | # <= 0: Reuse existing files (e.g., vocaburary, regions files). 22 | # <= Set this to 1 if existing files are obsolete or broken and causing errors. 23 | # <= Set this to 0 to avoid re-generating existing files. 24 | 25 | nm=s-dp # Data name 26 | idir=data # Where the input files (token/label files and label dictionary) are. 27 | bb=2 # Number of training data batches. 28 | p0=3 # Window size of text region embedding (1st layer), 29 | ss=0.25 # Step-size (learning rate) for the final supervised training. 30 | topdo=0.5 # Dropout rate applied to the input to the top layer. Default: No dropout. 31 | # 32 | nodes=250 # Number of feature maps. 33 | epochs=50 # 50/bb epochs 34 | dim=100 # Dimensionality of unsupervised embeddings. 35 | xmax=30 # Size of uni-gram vocabulary 36 | x3max=200 # Size of 1-3gram vocabulary for unsupervised embeddings 37 | trikw=tri${x3max}k # Used in the filenames for region files of 1-3grams. 38 | lays=5 # Number of hidden weight layers 39 | 40 | 41 | #------------------------------------- 42 | #--- Step 1. Train an unsupervised embedding function. 43 | #------------------------------------- 44 | p=5 # Region size 5 45 | n=1 # Use bag of word uni-grams to represent regions. 46 | pnm=p${p} # Used for generating filenames. 47 | 48 | #--- Unsupervised embedding training. 49 | dpcnn_gen_regions_unsemb; if [ $? != 0 ]; then echo $shnm: dpcnn_gen_regions_unsup failed.; exit 1; fi 50 | dpcnn_train_unsemb; if [ $? != 0 ]; then echo $shnm: dpcnn_train_unsemb failed.; exit 1; fi 51 | rm -f ${tmpdir}/${nm}-td-uns*${pnm} # clean up 52 | 53 | #--- Generate region file for a side layer for the final training. 54 | #---- NOTE: dpcnn_gen_regions_side must be called AFTER dpcnn_gen_regions_unsemb with the same parameters. 55 | dpcnn_gen_regions_side; if [ $? != 0 ]; then echo $shnm: dpcnn_gen_regions_side failed.; exit 1; fi 56 | 57 | 58 | #------------------------------------- 59 | #--- Step 2. Final training using unsupervised embeddings. 60 | #------------------------------------- 61 | #--- Prepare input files. 62 | p=$p0; pnm=p${p} 63 | dpcnn_gen_regions # prepare regions file for layer-0, target file, etc. 64 | if [ $? != 0 ]; then echo $shnm: dpcnn_gen_regions failed.; exit 1; fi 65 | 66 | #--- Training 67 | spnms=( p5 ) # One unsupervised embedding 68 | mynm=dpcnn-1uemb-${nm}-${lays}-p${p}-ss${ss}-${topdo} 69 | do_save=1 # Save models in files. 70 | # do_save=0 # Don't save models in files. 71 | dpcnn_training 72 | if [ $? != 0 ]; then echo dpcnn_training failed.; exit 1; fi 73 | 74 | 75 | #------------------------------------- 76 | #--- Step 3. Apply the model saved after $epochs epochs to test data. 77 | #------------------------------------- 78 | #--- Prediction values are written to: 79 | #--- ${outdir}/${mynm}.epo${tepo}.ReNet.pred.{txt|bin} 80 | epochs=45 81 | do_write_text=1 82 | dpcnn_predict 83 | if [ $? != 0 ]; then echo dpcnn_predict failed.; exit 1; fi 84 | -------------------------------------------------------------------------------- /examples/dpcnn-without-unsemb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #**************************************************************************** 3 | # Training/testing DPCNN without unsupervised embeddings. 4 | # 5 | # Step 0. Prepare input files. 6 | # Prepare token files and label files of training, validation, and test sets. 7 | # and a label dictionary file according to naming conventions as follows: 8 | # * Naming conventions for input files ( see data/s-dp.* for example) 9 | # - training data: ${nm}-td.${no}of${bb}.{txt.tok|cat} where bb is the number of batches, no=1,2,...,bb 10 | # - validation data: ${nm}-dv.{txt.tok|cat} 11 | # - test data: ${nm}-test.{txt.tok|cat} 12 | # - label dictionary:${nm}.catdic 13 | # 14 | # Step 1. Train DPCNN. 15 | # 16 | # Step 2. Apply a model to test data and write prediction values to a file. 17 | #**************************************************************************** 18 | source sh-common.sh # Constants and functions 19 | source dpcnn-functions.sh # Constants and functions for DPCNN 20 | 21 | max_jobs=5 # <= Data preparation (dpcnn_gen_regions*) creates up to this number of background processes. 22 | dont_reuse=1 # <= 1: Don't reuse old files and create new files. 23 | # <= 0: Reuse existing files (e.g., vocaburary, regions files). 24 | # <= Set this to 1 if existing files are obsolete or broken and causing errors. 25 | # <= Set this to 0 to avoid re-generating existing files. 26 | 27 | #------------------- 28 | #--- Step 0. Prepare input files (token files, label files, and label dictionary). 29 | #------------------- 30 | # - dpcnn-tokenize.sh can be used to produce token files and label files to prepare input files 31 | # from csv files of Yelp.p etc. 32 | # - Please follow the naming conventions above if you do it by yourself. 33 | # - Small toy data "s-dp" is provided at data/. 34 | 35 | #------------------- 36 | #--- Step 1. Train DPCNN 37 | #------------------- 38 | gpu=-1 # <= Change this to, e.g., "gpu=0" to use a specific GPU. 39 | idir=gendata # <= Change this to where the dataset files (tokenized text files and label files etc.) are. 40 | # <= Uncomment one of the following 6 lines. 41 | #--- 42 | nm=s-dp; bb=2; p=3; ss=0.25; topdo=0.5; tepo=50; idir=data # small sample data 43 | # nm=yelppol; bb=5; p=3; ss=0.05; topdo=0.5; tepo=135 44 | # nm=yelpfull; bb=5; p=3; ss=0.05; topdo=0.5; tepo=125 45 | # nm=yah10; bb=10; p=5; ss=0.05; topdo=0; tepo=135 46 | # nm=amafull; bb=10; p=3; ss=0.025; topdo=0; tepo=150 47 | # nm=amapol; bb=10; p=3; ss=0.025; topdo=0; tepo=145 48 | #--- 49 | # nm: Data name. 50 | # bb: Number of training data batches. 51 | # p: Window size of the text region embedding layer (1st layer). 52 | # ss: Step size (learning rate) 53 | # topdo: Dropout parameter applied to the input to the top (output) layer. 54 | # tepo (test epochs) was chosen based on the validation performance. 55 | # 56 | lays=15 # Number of weight layers. 57 | nodes=250 # Number of feature maps. 58 | epochs=150 # Train for 150/bb epochs (bb is the number of train data batches). 59 | # NOTE: Perform early stopping based on the validation performance 60 | # after the learning rate is reduced at epochs*4/5. 61 | xmax=30 # Vocabulary size / 1000 (e.g., 30 means 30K) 62 | pnm=p${p} # Used to generate input filenames. 63 | 64 | if [ "$nm" = "s-dp" ]; then epochs=50; lays=5; fi # Changing parameters for toy data. 65 | 66 | mynm=dpcnn-${nm}-${lays} # used to generate output filenames 67 | 68 | #--- Generate input files (regions files etc. ) 69 | dpcnn_gen_regions 70 | if [ $? != 0 ]; then echo dpcnn_gen_regions failed; exit 1; fi 71 | 72 | #--- Training without unsupervised embeddings. 73 | do_save=1 # Save models in files. 74 | # do_save=0 # Don't save models. 75 | dpcnn_training 76 | if [ $? != 0 ]; then echo dpcnn_training failed; exit 1; fi 77 | 78 | #--- Apply the model saved after $tepo epochs to test data and write prediction values to a file 79 | # do_write_text=0 # Write in the binary format, faster 80 | do_write_text=1 # Write in the text format, slower 81 | epochs=$tepo # Use the models saved after $tepo ecpochs. 82 | dpcnn_predict 83 | if [ $? != 0 ]; then echo dpcnn_predict failed; exit 1; fi 84 | -------------------------------------------------------------------------------- /examples/extract_text-imdb.pl: -------------------------------------------------------------------------------- 1 | #--- for categorization (not for counting cooccurrences) 2 | #--- replace

by space 3 | 4 | use strict 'vars'; 5 | my $arg_num = $#ARGV + 1; 6 | if ($arg_num != 5) { 7 | print STDERR "lst_fn inp_dir cat out_fn cat_fn\n"; 8 | exit; 9 | 10 | } 11 | 12 | my $argx = 0; 13 | my $lst_fn = $ARGV[$argx++]; 14 | my $inp_dir = $ARGV[$argx++]; 15 | my $cat = $ARGV[$argx++]; 16 | my $out_fn = $ARGV[$argx++]; 17 | my $cat_fn = $ARGV[$argx++]; 18 | 19 | open(LST, $lst_fn) or die("Can't open $lst_fn\n"); 20 | open(OUT, ">$out_fn") or die("Can't open $out_fn\n"); 21 | open(CAT, ">$cat_fn") or die("Can't open $cat_fn\n"); 22 | 23 | my $inc = 1000; 24 | my $count = 0; 25 | while() { 26 | my $line = $_; 27 | chomp $line; 28 | my $inp_fn = $inp_dir . $line; 29 | my $out = &proc($inp_fn); 30 | ++$count; 31 | 32 | print OUT "$out\n"; 33 | print CAT "$cat\n"; 34 | } 35 | print "$lst_fn ($count) ... \n"; 36 | 37 | close(OUT); 38 | close(LST); 39 | 40 | ###### just remove newline 41 | sub proc { 42 | my($inp_fn) = @_; 43 | 44 | open(INP, $inp_fn) or die("Can't open $inp_fn\n"); 45 | my $out = ""; 46 | while() { 47 | my $line = $_; 48 | chomp $line; 49 | $out .= "$line "; 50 | } 51 | close(INP); 52 | return $out; 53 | } 54 | -------------------------------------------------------------------------------- /examples/ld-extract.pl: -------------------------------------------------------------------------------- 1 | 2 | use strict 'vars'; 3 | my $arg_num = $#ARGV+1; 4 | if ($arg_num != 2 && $arg_num != 3) { 5 | print STDERR "inp_fn outnm [ do_dbpedia=\(0\|1\) ]\n"; 6 | exit -1; 7 | } 8 | 9 | my $argx = 0; 10 | my $inp_fn = $ARGV[$argx++]; 11 | my $outnm = $ARGV[$argx++]; 12 | my $do_dbpedia = 0; 13 | if ($argx < $arg_num) { $do_dbpedia = $ARGV[$argx++]; } 14 | 15 | open(INP, "$inp_fn") or die("Can't open $inp_fn\n"); 16 | my $txt_fn = $outnm . '.txt'; 17 | my $cat_fn = $outnm . '.cat'; 18 | my $catdic = $outnm . '.catdic'; 19 | open(TXT, ">$txt_fn") or die("Can't open $txt_fn\n"); 20 | open(CAT, ">$cat_fn") or die("Can't open $cat_fn\n"); 21 | open(CATDIC, ">$catdic") or die("Cant' open $catdic\n"); 22 | 23 | my %hash = (); 24 | 25 | while() { 26 | my $line = $_; 27 | chomp $line; 28 | my $exp = '^\"([^\"]+)\"\,\"(.*)\"$'; 29 | if ($do_dbpedia == 1) { $exp = '^([^\"]+)\,\"(.*)\"$'; } 30 | 31 | if ($line =~ /$exp/) { 32 | my $cat = $1; 33 | my $txt = $2; 34 | $txt = &cleanup($txt); 35 | if ($hash{$cat} != 1) { 36 | $hash{$cat} = 1; 37 | print CATDIC "$cat\n"; 38 | } 39 | print TXT "$txt\n"; 40 | print CAT "$cat\n"; 41 | } 42 | else { 43 | print STDERR "unexpected line: $line\n"; 44 | } 45 | } 46 | 47 | close(INP); 48 | close(TXT); 49 | close(CAT); 50 | close(CATDIC); 51 | 52 | ##### 53 | sub cleanup { 54 | my($inp) = @_; 55 | 56 | my $out = $inp; 57 | 58 | $out =~ s/\"\,\"/ \| /gs; # delimiter between the summary and the text 59 | $out =~ s/\\\"\"/\"/gs; 60 | $out =~ s/\"\"/\"/gs; 61 | $out =~ s/\\n/ /gs; 62 | $out =~ s/\s/ /gs; 63 | return $out; 64 | } -------------------------------------------------------------------------------- /examples/ld-functions.sh: -------------------------------------------------------------------------------- 1 | #*************************************************************************** 2 | # Functions below were used to extract and tokenize text used in [JZ17]. 3 | # They are not meant for general purposes. 4 | # 5 | # NOTE: "source sh-common.sh" is needed before "source dpcnn-functions.sh". 6 | # [JZ17] Deep pyramid convolutional neural networks for text categorization 7 | #*************************************************************************** 8 | 9 | #--- Extract text and labels. 10 | #--- Input: ${orgdir}/train.csv, ${orgdir}/test.csv # Yelp.{f|p},Yahoo,Ama.{f|p},AG,Sogou,Dbpedia 11 | #--- # donwloaded 12 | #--- 13 | #--- Output:${odir}/${nm}-{train|test}.{txt|cat} # training/test text/label files 14 | #--- ${odir}/${nm}.catdic # label dictionary 15 | ld_extract () { 16 | local fnm=ld_extract 17 | 18 | if [ "$orgdir" = "" ]; then echo $fnm: orgdir is missing.; return 1; fi 19 | if [ "$odir" = "" ]; then echo $fnm: odir is missing.; return 1; fi 20 | if [ "$nm" = "" ]; then echo $fnm: nm is missing.; return 1; fi 21 | 22 | echo $fnm: orgdir=$orgdir odir=$odir nm=$nm 23 | 24 | #--- Extract text and labels 25 | echo; echo Extracting text and labels ... 26 | local typ; for typ in train test; do 27 | perl ld-extract.pl ${orgdir}/${typ}.csv ${odir}/${nm}-${typ} $do_dbpedia 28 | if [ $? != 0 ]; then echo $fnm: text extraction failed.; return 1; fi 29 | done 30 | 31 | #--- Prepare a label dictionary 32 | rm -f ${odir}/${nm}-train.catdic 33 | sort < ${odir}/${nm}-test.catdic > ${odir}/${nm}.catdic 34 | if [ $? != 0 ]; then echo $fnm: sort failed.; return 1; fi 35 | rm -f ${odir}/${nm}-test.catdic 36 | 37 | return 0 38 | } 39 | 40 | #--- Split training data into the training portion and the validation portion. 41 | #--- Split the training portion into data batches. 42 | #--- Tokenize training, validation, and test data. 43 | #--- Optionally, extract text and labels from a csv file in the charcnn format. 44 | #--- 45 | #--- Input: ${orgdir}/train.csv, ${orgdir}/test.csv 46 | #--- Output: ${odir}/${nm}-td.${batch_id}.{txt.tok|cat} where batch_id is 1of${bb}, 2of${bb}, ... 47 | #--- ${odir}/${nm}-dv.{txt.tok|cat} 48 | #--- ${odir}/${nm}-test.${txt.tok|cat} 49 | ld_split_tokenize () { 50 | local fnm=ld_split_tokenize 51 | if [ "$orgdir" = "" ]; then echo $fnm: orgdir is missing.; return 1; fi 52 | if [ "$odir" = "" ]; then echo $fnm: odir is missing.; return 1; fi 53 | if [ "$bb" = "" ]; then echo $fnm: bb is missing.; return 1; fi 54 | if [ "$nm" = "" ]; then echo $fnm: nm is missing.; return 1; fi 55 | if [ "$num" = "" ]; then echo $fnm: num is missing.; return 1; fi 56 | if [ "$dev" = "" ]; then dev=10; fi 57 | 58 | # optional: max_jobs, do_dbpedia, reg 59 | if [ "$reg" = "" ]; then reg=${odir}/empty; touch $reg; fi 60 | if [ "$max_jobs" = "" ]; then max_jobs=1; fi 61 | echo $fnm: orgdir=$orgdir odir=$odir bb=$bb nm=$nm num=$num dev=$dev reg=$reg 62 | echo $fnm: max_jobs=$max_jobs do_dbpedia=$do_dbpedia 63 | 64 | #--- Extract text and labels 65 | ld_extract 66 | if [ $? != 0 ]; then echo $fnm: ld_extract failed.; return 1; fi 67 | _onm=${odir}/${nm} 68 | 69 | #--- Split the training set ("train") into a developemnt set ("dv") and the rest ("td") 70 | echo Splitting ... 71 | local ext; for ext in txt cat; do 72 | local _tmp=${_onm}.tmp 73 | $prep_exe split_text input_fn=${_onm}-train.${ext} random_seed=1 output_fn_stem=${_tmp}.${ext} split=$((num-dev)):${dev} 74 | if [ $? != 0 ]; then echo $fnm: split_text failed.; return 1; fi 75 | mv ${_tmp}.${ext}.1of2 ${_onm}-td.${ext} 76 | mv ${_tmp}.${ext}.2of2 ${_onm}-dv.${ext} 77 | done 78 | 79 | #--- Split the td set (used as training data) into $bb batches 80 | local _split=1; local no; for (( no=1; no<=bb-1; no++ )) do _split=${_split}:1; done # e.g., 1:1:1:1:1 for bb=5 81 | echo $fnm: _split=$_split 82 | local ext; for ext in txt cat; do 83 | if [ "$bb" = 1 ]; then 84 | mv ${_onm}-td.${ext} ${_onm}-td.${ext}.1of1 85 | else 86 | $prep_exe split_text input_fn=${_onm}-td.${ext} random_seed=1 output_fn_stem=${_onm}-td.${ext} split=$_split 87 | if [ $? != 0 ]; then echo $fnm: split_text failed.; return 1; fi 88 | fi 89 | done 90 | 91 | 92 | echo Tokenizing ... 93 | _cmd_=(); local num=0 94 | #--- Tokenize dv and test. 95 | local set; for set in dv test; do 96 | _cmd_[$num]="perl to_tokens.pl ${_onm}-${set}.txt $reg .tok 1"; num=$((num+1)) 97 | done 98 | #--- Tokenize td. 99 | local no; for (( no=1; no<=bb; no++ )); do 100 | _cmd_[$num]="perl to_tokens.pl ${_onm}-td.txt.${no}of${bb} $reg .tok 1"; num=$((num+1)) 101 | done 102 | do_jobs $max_jobs $num tok-${nm} 103 | if [ $? != 0 ]; then echo $nm: do_jobs failed.; return 1; fi 104 | 105 | echo Done ... 106 | 107 | #--- Rename td tokenized files 108 | local no; for (( no=1; no<=bb; no++ )) do 109 | mv ${_onm}-td.txt.${no}of${bb}.tok ${_onm}-td.${no}of${bb}.txt.tok 110 | mv ${_onm}-td.cat.${no}of${bb} ${_onm}-td.${no}of${bb}.cat 111 | done 112 | 113 | return 0 114 | } 115 | -------------------------------------------------------------------------------- /examples/lstm-2unsemb-imdb-elec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #--- LSTM with two unsupervised embeddings (LstmF and LstmB) 3 | 4 | #-----------------# 5 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 6 | mem=4 # pre-allocate 4GB device memory 7 | source sh-common.sh 8 | #-----------------# 9 | nm=imdb; supepo=100; u_stepsize=0.5; mb=50 10 | # nm=elec; supepo=50; u_stepsize=0.25; mb=100 # Uncomment this to train/test on Elec. 11 | 12 | ddir=data 13 | udir=for-semi # <= Where unlabeled data is. Downloaded if udir=for-semi. 14 | 15 | options="LowerCase UTF8" 16 | catdic=${ddir}/${nm}_cat.dic 17 | z=l2 # to avoid name conflict with other scripts 18 | dim=100 # dimensionality of unsupervised embeddings. 19 | unsepo=30 # number of epochs for unsupervised embedding training. 20 | 21 | #--- Download unlabeled data. 22 | uns_lst=${tmpdir}/${nm}${z}-trnuns.lst # unlabeled data 23 | if [ "$nm" = "imdb" ]; then 24 | echo data/imdb-train.txt.tok > $uns_lst 25 | echo ${udir}/imdb-unlab.txt.tok >> $uns_lst 26 | find_file $udir imdb-unlab.txt.tok; if [ $? != 0 ]; then echo $shnm: find_file failed.; exit 1; fi 27 | elif [ "$nm" = "elec" ]; then 28 | echo ${udir}/elec-25k-unlab00.txt.tok > $uns_lst 29 | echo ${udir}/elec-25k-unlab01.txt.tok >> $uns_lst 30 | find_file $udir elec-25k-unlab00.txt.tok; if [ $? != 0 ]; then echo $shnm: find_file failed.; exit 1; fi 31 | find_file $udir elec-25k-unlab01.txt.tok; if [ $? != 0 ]; then echo $shnm: find_file failed.; exit 1; fi 32 | else 33 | echo Unexpected dataset name: $nm; exit 1 34 | fi 35 | 36 | #*** Embedding learning on unlabeled data 37 | #--- vocabulary for X (features) 38 | xvoc=${tmpdir}/${nm}${z}-trn.vocab 39 | $prep_exe gen_vocab input_fn=${ddir}/${nm}-train.txt.tok vocab_fn=$xvoc $options WriteCount max_vocab_size=30000 40 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 41 | #--- NOTE: With larger unlabeled data, try input_fn=$uns_lst and max_vocab_size=100000. 42 | 43 | #--- vocabulary for Y (target) 44 | yvoc=${tmpdir}/${nm}${z}-minstop-uns.vocab 45 | stop_fn=data/minstop.txt # function words 46 | $prep_exe gen_vocab $options input_fn=$uns_lst vocab_fn=$yvoc WriteCount max_vocab_size=30000 stopword_fn=$stop_fn 47 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 48 | 49 | for unslay in LstmF LstmB; do # Forward and backward 50 | #--- Generate input data for unsupervised embedding learning 51 | if [ "$unslay" = "LstmF" ]; then # forward (left to right) 52 | lr=RightOnly # predict the words on the right 53 | lrkw=F 54 | fi 55 | if [ "$unslay" = "LstmB" ]; then # backward (right to left) 56 | lr=LeftOnly # predict words on the left 57 | lrkw=B 58 | fi 59 | 60 | unsnm=${nm}${z}-uns-${lrkw} 61 | $prep_exe gen_regions_unsup $options x_type=Bow input_fn=$uns_lst $lr \ 62 | x_vocab_fn=$xvoc y_vocab_fn=$yvoc region_fn_stem=${tmpdir}/${unsnm} \ 63 | patch_size=1 patch_stride=1 padding=0 dist=5 \ 64 | x_ext=.xsmatbcvar y_ext=.ysmatbcvar NoSkip 65 | if [ $? != 0 ]; then echo $shnm: gen_regions_unsup failed.; exit 1; fi 66 | 67 | #--- Embedding training on unlabeled data 68 | mynm=${nm}-Lstm${lrkw}-dim${dim} 69 | lay_fn=${outdir}/${mynm} 70 | logfn=${logdir}/${mynm}.log 71 | echo Training unsupervised embedding ... see $logfn 72 | $exe $gpu:$mem train top_CountRegions NoTest Regression NoCusparseIndex 0save_layer_fn=$lay_fn \ 73 | inc=5000 trnname=$unsnm data_dir=$tmpdir \ 74 | reg_L2=0 step_size=$u_stepsize num_epochs=$unsepo \ 75 | layers=1 loss=Square mini_batch_size=10 momentum=0.9 random_seed=1 \ 76 | datatype=sparse x_ext=.xsmatbcvar y_ext=.ysmatbcvar \ 77 | 0layer_type=$unslay 0nodes=$dim 0chop_size=50 \ 78 | zero_Y_ratio=10 zero_Y_weight=1 > $logfn 79 | if [ $? != 0 ]; then echo $shnm: Embedding training failed.; exit 1; fi 80 | done 81 | 82 | #*** Generate input data for the final supervised training 83 | for set in train test; do 84 | rnm=${tmpdir}/${nm}${z}-${set}-p1 85 | $prep_exe gen_regions $options NoSkip region_fn_stem=$rnm \ 86 | input_fn=${ddir}/${nm}-${set} vocab_fn=$xvoc label_dic_fn=$catdic \ 87 | patch_size=1 88 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 89 | done 90 | 91 | #*** Training with labeled data 92 | fnF=${outdir}/${nm}-LstmF-dim${dim}.epo${unsepo}.ReLayer0 93 | fnB=${outdir}/${nm}-LstmB-dim${dim}.epo${unsepo}.ReLayer0 94 | 95 | mynm=lstm-2unsemb-${nm}; logfn=${logdir}/${mynm}.log; csvfn=${csvdir}/${mynm}.csv 96 | echo; echo Training with labeled data and unsupervised embeddings ... see $logfn and $csvfn 97 | suplam=0; suplamtop=1e-4 # regularization parameters. 98 | $exe $gpu:$mem train num_sides=2 0side0_layer_type=LstmF 0side0_layer_fn=$fnF 0side1_layer_type=LstmB 0side1_layer_fn=$fnB \ 99 | NoGate_i NoGate_o top_dropout=0.5 top_reg_L2=$suplamtop test_mini_batch_size=500 \ 100 | max_loss=5 inc=5000 trnname=${nm}${z}-train-p1 tstname=${nm}${z}-test-p1 data_dir=$tmpdir \ 101 | test_interval=25 reg_L2=$suplam step_size=1 evaluation_fn=$csvfn \ 102 | layers=2 loss=Square mini_batch_size=$mb momentum=0.9 random_seed=1 \ 103 | datatype=sparse \ 104 | num_epochs=$supepo ss_scheduler=Few ss_decay=0.1 ss_decay_at=$((supepo*4/5)) \ 105 | 0layer_type=Lstm2 0nodes=500 0chop_size=50 \ 106 | 1layer_type=Pooling 1num_pooling=1 1pooling_type=Max > $logfn 107 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 108 | 109 | rm -f ${tmpdir}/${nm}${z}* -------------------------------------------------------------------------------- /examples/lstm-5unsemb-imdb-elec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #--- Semi-supervised LSTM with 2 unsupervised LSTM embeddings 3 | #--- and 3 unsupervised CNN embeddings. 4 | 5 | #-----------------# 6 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 7 | mem=4 # pre-allocate 4GB device memory 8 | source sh-common.sh 9 | #-----------------# 10 | nm=imdb; mb=50; supite=100 11 | # nm=elec; mb=100; supite=50 # Uncomment this to train/test on Elec. 12 | 13 | ddir=data 14 | lstmdir=for-semi # <= Change this to where LSTM embeddings are. Downloaded if lstmdir=for-semi. 15 | cnndir=for-semi # <= Change this to where CNN embeddings are. Downloaded if cnndir=for-semi. 16 | ##### 17 | ##### WARNING: If your system uses Big Endian (Motorola convention), you cannot use the 18 | ##### downloaded files! They are in the little-endian format (Intel convention)! 19 | ##### 20 | 21 | options="LowerCase UTF8" 22 | catdic=${ddir}/${nm}_cat.dic 23 | z=l5 # to avoid name conflict with other scripts 24 | dim=100 # dimensionality of unsupervised embeddings. 25 | 26 | #--- Download unsupervised embedding files if required. 27 | lay_fn0=${nm}-LstmF-dim${dim}.lay.epo30.ReLayer0 28 | lay_fn1=${nm}-LstmB-dim${dim}.lay.epo30.ReLayer0 29 | for fn in $lay_fn0 $lay_fn1; do find_file $lstmdir $fn; if [ $? != 0 ]; then echo $shnm: find_file failed.; exit 1; fi; done 30 | lay_fn0=${lstmdir}/${lay_fn0}; lay_fn1=${lstmdir}/${lay_fn1} 31 | 32 | lay_fn2=${nm}-uns-p5.dim${dim}.epo10.ReLayer0 33 | lay_fn3=${nm}-unsx3-p5.dim${dim}.epo10.ReLayer0 34 | lay_fn4=${nm}-parsup-p3p5.dim${dim}.epo10.ReLayer0 35 | for fn in $lay_fn2 $lay_fn3 $lay_fn4; do find_file $cnndir $fn; if [ $? != 0 ]; then echo $shnm: find_file failed.; exit 1; fi; done 36 | lay_fn2=${cnndir}/${lay_fn2}; lay_fn3=${cnndir}/${lay_fn3}; lay_fn4=${cnndir}/${lay_fn4} 37 | 38 | #--- Generate region files for unsupervised LSTM embeddings 39 | voc01=${tmpdir}/${nm}${z}-01.wmap 40 | $exe $gpu write_word_mapping layer_type=LstmF layer0_fn=$lay_fn0 word_map_fn=$voc01 41 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 42 | 43 | for set in train test; do 44 | rnm=${tmpdir}/${nm}${z}-${set}-p1 45 | $prep_exe gen_regions $options NoSkip region_fn_stem=$rnm \ 46 | input_fn=${ddir}/${nm}-${set} vocab_fn=$voc01 label_dic_fn=$catdic \ 47 | patch_size=1 48 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 49 | done 50 | 51 | #--- Generate region files for unsupervised CNN embeddings 52 | wm2=${tmpdir}/${nm}${z}-2.wmap 53 | $exe $gpu write_word_mapping layer0_fn=$lay_fn2 layer_type=Weight+ word_map_fn=$wm2 54 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 55 | 56 | wm3=${tmpdir}/${nm}${z}-3.wmap 57 | $exe $gpu write_word_mapping layer0_fn=$lay_fn3 layer_type=Weight+ word_map_fn=$wm3 58 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 59 | 60 | wm4=${tmpdir}/${nm}${z}-4.wmap 61 | $exe $gpu write_word_mapping layer0_fn=$lay_fn4 layer_type=Weight+ word_map_fn=$wm4 62 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 63 | 64 | for set in train test; do 65 | for no in 2 3 4; do 66 | p=5 67 | rnm=${tmpdir}/${nm}${z}-${set}-${no}-p${p}bow 68 | $prep_exe gen_regions $options NoSkip Bow RegionOnly region_fn_stem=$rnm \ 69 | input_fn=${ddir}/${nm}-${set} vocab_fn=${tmpdir}/${nm}${z}-${no}.wmap \ 70 | patch_size=$p padding=$(((p-1)/2)) 71 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 72 | done 73 | done 74 | 75 | #--- Training with labeled data and five unsupervised embeddings 76 | mynm=lstm-5unsemb-${nm}-dim${dim} 77 | logfn=${logdir}/${mynm}.log; csvfn=${csvdir}/${mynm}.csv 78 | echo; echo Training with labeled data and five tv-embeddings ... see $logfn and $csvfn 79 | suplam=0; suplamtop=1e-4 # regularization parameters. 80 | $exe $gpu:$mem train reg_L2=$suplam top_reg_L2=$suplamtop top_dropout=0.5 \ 81 | num_sides=5 0side0_layer_type=LstmF 0side0_layer_fn=$lay_fn0 0side1_layer_type=LstmB 0side1_layer_fn=$lay_fn1 \ 82 | 0side2_layer_type=Weight+ 0side2_layer_fn=$lay_fn2 0side2_dsno=1 \ 83 | 0side3_layer_type=Weight+ 0side3_layer_fn=$lay_fn3 0side3_dsno=2 \ 84 | 0side4_layer_type=Weight+ 0side4_layer_fn=$lay_fn4 0side4_dsno=3 \ 85 | NoGate_i NoGate_o test_mini_batch_size=500 \ 86 | max_loss=5 inc=5000 trnname=${nm}${z}-train- tstname=${nm}${z}-test- data_dir=${tmpdir} \ 87 | test_interval=25 step_size=1 evaluation_fn=$csvfn \ 88 | layers=2 loss=Square mini_batch_size=$mb momentum=0.9 random_seed=1 \ 89 | datatype=sparse dsno0=p1 dsno1=2-p5bow dsno2=3-p5bow dsno3=4-p5bow \ 90 | num_epochs=$supite ss_scheduler=Few ss_decay=0.1 ss_decay_at=$((supite*4/5)) \ 91 | 0layer_type=Lstm2 0nodes=500 0chop_size=50 \ 92 | 1layer_type=Pooling 1num_pooling=1 1pooling_type=Max > $logfn 93 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 94 | 95 | rm -f ${tmpdir}/${nm}${z}* -------------------------------------------------------------------------------- /examples/lstm-imdb-elec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #--- LSTM 3 | #-----------------# 4 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 5 | mem=4 # pre-allocate 2GB device memory 6 | source sh-common.sh 7 | #-----------------# 8 | nm=imdb; ite=100; ite1=80; mb=50 9 | # nm=elec; ite=50; ite1=40; mb=100 # <= Uncomment this to train/test on Elec. 10 | 11 | ddir=data 12 | options="LowerCase UTF8" 13 | catdic=${ddir}/${nm}_cat.dic 14 | z=l1 # to avoid name conflict with other scripts 15 | 16 | #--- vocabulary 17 | xvoc=${tmpdir}/${nm}${z}-trn.vocab 18 | $prep_exe gen_vocab $options input_fn=${ddir}/${nm}-train.txt.tok vocab_fn=$xvoc WriteCount max_vocab_size=30000 19 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 20 | 21 | #--- Generate input data for the final supervised training 22 | for set in train test; do 23 | rnm=${tmpdir}/${nm}${z}-${set}-p1 24 | $prep_exe gen_regions $options NoSkip region_fn_stem=$rnm \ 25 | input_fn=${ddir}/${nm}-${set} vocab_fn=$xvoc label_dic_fn=$catdic \ 26 | patch_size=1 27 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 28 | done 29 | 30 | #--- Training 31 | mynm=lstm-${nm} 32 | logfn=${logdir}/${mynm}.log; csvfn=${csvdir}/${mynm}.csv 33 | echo; echo Training ... see $logfn and $csvfn 34 | $exe $gpu:$mem train NoGate_i NoGate_o top_dropout=0.5 test_mini_batch_size=500 \ 35 | max_loss=5 inc=5000 trnname=${nm}${z}-train-p1 tstname=${nm}${z}-test-p1 data_dir=$tmpdir \ 36 | test_interval=25 reg_L2=0 step_size=1 evaluation_fn=$csvfn \ 37 | layers=2 loss=Square mini_batch_size=$mb momentum=0.9 random_seed=1 \ 38 | datatype=sparse \ 39 | num_epochs=$ite ss_scheduler=Few ss_decay=0.1 ss_decay_at=$ite1 \ 40 | 0layer_type=Lstm2 0nodes=500 0chop_size=50 \ 41 | 1layer_type=Pooling 1num_pooling=1 1pooling_type=Max > $logfn 42 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 43 | 44 | rm -f ${tmpdir}/${nm}${z}* -------------------------------------------------------------------------------- /examples/merge_dic.pl: -------------------------------------------------------------------------------- 1 | use strict 'vars'; 2 | 3 | my $arg_num = $#ARGV+1; 4 | if ($arg_num < 2) { 5 | print STDERR "max_num dic1 dic2 ... \n"; 6 | exit -1; 7 | } 8 | my $argx = 0; 9 | my $max_num = $ARGV[$argx++]; 10 | 11 | my %hash = (); 12 | for ( ; $argx < $arg_num; ++$argx) { 13 | my $fn = $ARGV[$argx]; 14 | open(INP, $fn) or die("Can't open $fn\n"); 15 | while() { 16 | my $line = $_; 17 | chomp $line; 18 | if ($line =~ /^(\S.*)\t(\d+)$/) { 19 | my $word = $1; 20 | my $count = $2; 21 | if ($hash{$word} > 0) { 22 | print STDERR "duplicated word: $word\n"; 23 | exit -1; 24 | } 25 | $hash{$word} = $count; 26 | } 27 | elsif ($line =~ /^(\S.*)$/) { 28 | my $word = $1; 29 | my $count = 1; 30 | if ($hash{$word} > 0) { 31 | print STDERR "duplicated word: $word\n"; 32 | exit -1; 33 | } 34 | $hash{$word} = $count; 35 | } 36 | else { 37 | print STDERR "invalid line: $line\n"; 38 | exit -1; 39 | } 40 | } 41 | close(INP); 42 | } 43 | 44 | my $num = 0; 45 | foreach my $word (sort { $hash{$b} <=> $hash{$a} } keys %hash) { 46 | if ($max_num > 0 && $num >= $max_num) { 47 | last; 48 | } 49 | my $count = $hash{$word}; 50 | print "$word\t$count\n"; 51 | ++$num; 52 | } 53 | exit 0 -------------------------------------------------------------------------------- /examples/other-sh/lstm-5unsemb-rcv1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #--- RCV1: semi-supervised LSTM with 2 unsupervised LSTM embeddings 3 | #--- and 3 unsupervised CNN embeddings. 4 | #--- NOTE: 5GB or more GPU device memory is required. 5 | 6 | #-----------------# 7 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 8 | mem=5 # pre-allocate 5GB device memory 9 | source sh-common.sh 10 | #-----------------# 11 | ddir=data 12 | ldir=rcv1_data # <= Change this to where RCV1 labeled data are 13 | lstmdir=for-semi # <= Change this to where LSTM embeddings are. Downloaded if lstmdir=for-semi. 14 | cnndir=for-semi # <= Change this to where CNN embeddings are. Downloaded if cnndir=for-semi. 15 | ##### 16 | ##### WARNING: If your system uses Big Endian (Motorola convention), you cannot use the 17 | ##### downloaded files! They are in the little-endian format (Intel convention)! 18 | ##### 19 | 20 | options="LowerCase UTF8" 21 | txt_ext=.txt.tok 22 | 23 | catdic=${ddir}/rcv1-lvl2.catdic 24 | z=l5 # to avoid name conflict with other scripts 25 | 26 | dim=300 27 | unsite=50; supite=100; supite1=80 28 | 29 | suplam=0; suplamtop=0 30 | 31 | #--- Prepare unsupervised embedding files. 32 | lay_fn0=rcv1-LstmF-dim${dim}.lay.epo${unsite}.ReLayer0 33 | lay_fn1=rcv1-LstmB-dim${dim}.lay.epo${unsite}.ReLayer0 34 | for fn in $lay_fn0 $lay_fn1; do find_file $lstmdir $fn; if [ $? != 0 ]; then echo $shnm: find_file failed.; exit 1; fi; done 35 | lay_fn0=${lstmdir}/${lay_fn0}; lay_fn1=${lstmdir}/${lay_fn1} 36 | 37 | lay_fn2=rcv1-uns-p20.dim100.epo10.ReLayer0 38 | lay_fn3=rcv1-unsx3-p20.dim100.epo10.ReLayer0 39 | lay_fn4=rcv1-parsup-p20p20.dim100.epo10.ReLayer0 40 | for fn in $lay_fn2 $lay_fn3 $lay_fn4; do find_file $cnndir $fn; if [ $? != 0 ]; then echo $shnm: find_file failed.; exit 1; fi; done 41 | lay_fn2=${cnndir}/${lay_fn2}; lay_fn3=${cnndir}/${lay_fn3}; lay_fn4=${cnndir}/${lay_fn4} 42 | 43 | #--- Generate input data for unsupervised LSTM embeddings 44 | voc01=${tmpdir}/rcv1${z}-01.wmap 45 | $exe $gpu write_word_mapping layer_type=LstmF layer0_fn=$lay_fn0 word_map_fn=$voc01 46 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 47 | 48 | for set in train test; do 49 | rnm=${tmpdir}/rcv1${z}-${set}-p1 50 | $prep_exe gen_regions NoSkip \ 51 | region_fn_stem=$rnm input_fn=${ldir}/rcv1-1m-${set} vocab_fn=$voc01 \ 52 | $options text_fn_ext=$txt_ext label_fn_ext=.lvl2 \ 53 | label_dic_fn=$catdic \ 54 | patch_size=1 patch_stride=1 padding=0 55 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 56 | done 57 | 58 | #--- Generate input data for unsupervised CNN embeddings 59 | wm2=${tmpdir}/rcv1${z}-2.wmap 60 | $exe $gpu write_word_mapping layer0_fn=$lay_fn2 layer_type=Weight+ word_map_fn=$wm2 61 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 62 | 63 | wm3=${tmpdir}/rcv1${z}-3.wmap 64 | $exe $gpu write_word_mapping layer0_fn=$lay_fn3 layer_type=Weight+ word_map_fn=$wm3 65 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 66 | 67 | wm4=${tmpdir}/rcv1${z}-4.wmap 68 | $exe $gpu write_word_mapping layer0_fn=$lay_fn4 layer_type=Weight+ word_map_fn=$wm4 69 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 70 | 71 | 72 | for set in train test; do 73 | for no in 2 3 4; do 74 | p=21 # b/c we want an odd number here ... 75 | rnm=${tmpdir}/rcv1${z}-${set}-${no}-p${p}bow 76 | $prep_exe gen_regions NoSkip Bow \ 77 | region_fn_stem=$rnm input_fn=${ldir}/rcv1-1m-${set} vocab_fn=${tmpdir}/rcv1${z}-${no}.wmap \ 78 | $options text_fn_ext=$txt_ext RegionOnly \ 79 | patch_size=$p patch_stride=1 padding=$(((p-1)/2)) 80 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 81 | done 82 | done 83 | 84 | #--- Training with labeled data and five unsupervised embeddings 85 | mynm=lstm-5unsemb-rcv1-dim${dim} 86 | logfn=${logdir}/${mynm}.log 87 | csvfn=${csvdir}/${mynm}.csv 88 | echo Training with labeled data and five unsupervised embeddings ... see $logfn and $csvfn 89 | $exe $gpu:$mem train reg_L2=$suplam top_reg_L2=$suplamtop top_dropout=0.5 \ 90 | num_sides=5 0side0_layer_type=LstmF 0side0_layer_fn=$lay_fn0 0side1_layer_type=LstmB 0side1_layer_fn=$lay_fn1 \ 91 | 0side2_layer_type=Weight+ 0side2_layer_fn=$lay_fn2 0side2_dsno=1 \ 92 | 0side3_layer_type=Weight+ 0side3_layer_fn=$lay_fn3 0side3_dsno=2 \ 93 | 0side4_layer_type=Weight+ 0side4_layer_fn=$lay_fn4 0side4_dsno=3 \ 94 | NoGate_i NoGate_o test_mini_batch_size=500 \ 95 | max_loss=5 inc=5000 trnname=rcv1${z}-train- tstname=rcv1${z}-test- data_dir=${tmpdir} \ 96 | test_interval=25 step_size=1 evaluation_fn=$csvfn \ 97 | layers=2 loss=Square mini_batch_size=50 momentum=0.9 random_seed=1 \ 98 | datatype=sparse dsno0=p1 dsno1=2-p21bow dsno2=3-p21bow dsno3=4-p21bow \ 99 | num_epochs=$supite ss_scheduler=Few ss_decay=0.1 ss_decay_at=$supite1 \ 100 | 0layer_type=Lstm2 0nodes=500 0chop_size=50 \ 101 | 1layer_type=Pooling 1num_pooling=1 1pooling_type=Max > $logfn 102 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 103 | 104 | rm -f ${tmpdir}/rcv1${z}* 105 | -------------------------------------------------------------------------------- /examples/other-sh/lstm-rcv1-20ng.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #--- RCV1: Supervised LSTM 3 | #-----------------# 4 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 5 | mem=4 # pre-allocate 2GB device memory 6 | source sh-common.sh 7 | #-----------------# 8 | nm=rcv1-1m; ldir=rcv1_data; epo=25; catdic=data/rcv1-lvl2.catdic; lab_ext=.lvl2; pl=Avg 9 | # nm=20ng; ldir=20ng_data; epo=50; catdic=${ldir}/20ng.catdic; lab_ext=.cat; pl=Max; prepopt=NoSkip 10 | 11 | options="LowerCase UTF8" 12 | txt_ext=.txt.tok 13 | 14 | z=l1 # to avoid name conflict with other scripts 15 | 16 | #--- vocabulary 17 | xvoc=${tmpdir}/${nm}${z}-trn.vocab 18 | $prep_exe gen_vocab input_fn=${ldir}/${nm}-train vocab_fn=$xvoc $options WriteCount text_fn_ext=$txt_ext max_vocab_size=30000 19 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 20 | 21 | #--- Generate input data for the final supervised training 22 | for set in train test; do 23 | rnm=${tmpdir}/${nm}${z}-${set}-p1 24 | $prep_exe gen_regions region_fn_stem=$rnm input_fn=${ldir}/${nm}-${set} vocab_fn=$xvoc \ 25 | $options text_fn_ext=$txt_ext label_fn_ext=$lab_ext \ 26 | label_dic_fn=$catdic $prepopt \ 27 | patch_size=1 patch_stride=1 padding=0 28 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 29 | done 30 | 31 | #--- Training 32 | mynm=lstm-${nm} 33 | logfn=${logdir}/${mynm}.log 34 | csvfn=${csvdir}/${mynm}.csv 35 | echo Training ... see $logfn and $csvfn 36 | num_pool=10 37 | $exe $gpu:$mem train optim=Rmsp NoGate_i NoGate_o top_dropout=0.5 \ 38 | max_loss=5 inc=5000 trnname=${nm}${z}-train-p1 tstname=${nm}${z}-test-p1 data_dir=$tmpdir \ 39 | test_interval=25 reg_L2=0 step_size=0.25e-3 evaluation_fn=$csvfn \ 40 | layers=3 loss=Square mini_batch_size=50 test_mini_batch_size=500 random_seed=1 \ 41 | datatype=sparse \ 42 | num_epochs=$epo ss_scheduler=Few ss_decay=0.1 ss_decay_at=$((epo*4/5)) \ 43 | 0layer_type=Lstm2 0nodes=500 0chop_size=50 \ 44 | 1layer_type=Pooling 1num_pooling=$num_pool 1pooling_type=$pl \ 45 | 2layer_type=Patch 2patch_size=$num_pool > $logfn 46 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 47 | 48 | rm -f ${tmpdir}/${nm}${z}* -------------------------------------------------------------------------------- /examples/other-sh/prep_elec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #### This script does data preprocessing for the Elec experiments with 25K training data points. 3 | #### You don't have to run this script since the preprocessed files are included in the package. 4 | 5 | #--- Step 1. Download data 6 | txtdir=../elec 7 | 8 | #--- Step 3. Generate token files (data/*.tok; one review per line and tokens are separated by white space) 9 | echo Generating token files ... 10 | 11 | dir=data 12 | cp -p ${txtdir}/elec-25k-train.txt ${dir}/ 13 | cp -p ${txtdir}/elec-test.txt ${dir}/ 14 | 15 | perl to_tokens.pl ${dir}/elec-25k-train.txt data/imdb_registered2.txt .tok & 16 | perl to_tokens.pl ${dir}/elec-test.txt data/imdb_registered2.txt .tok & 17 | wait 18 | echo Done ... 19 | 20 | -------------------------------------------------------------------------------- /examples/other-sh/prep_imdb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #### This script does data preprocessing for the IMDB experiments. 3 | #### You don't have to run this script since the preprocessed files are included in the package. 4 | 5 | #--- Step 1. Donwload aclImdb_v1.tar.gz from http://ai.stanford.edu/~amaas/data/sentiment/ and unzip it. 6 | 7 | #--- Step 2. Generate text files (data/*.txt; one review per line) and label files (data/*.cat; one review per line). 8 | orgdir=aclImdb # from aclImdb_v1.tar.gz 9 | for t in train test; do 10 | for pn in pos neg; do 11 | cat=$pn 12 | nm=imdb-${t}-${pn} 13 | where=${orgdir}/${t}/${pn} 14 | perl extract_text-imdb.pl lst/${nm}.lst ${where}/ $cat data/${nm}.txt data/${nm}.cat 15 | done 16 | done 17 | 18 | cat_dic=data/imdb_cat.dic # list of labels 19 | echo neg > $cat_dic 20 | echo pos >> $cat_dic 21 | 22 | #--- Step 3. Generate token files (data/*.tok; one review per line and tokens are separated by white space) 23 | echo Generating token files ... 24 | perl to_tokens.pl data/imdb-train-pos.txt data/imdb_registered2.txt .tok & 25 | perl to_tokens.pl data/imdb-train-neg.txt data/imdb_registered2.txt .tok & 26 | perl to_tokens.pl data/imdb-test-pos.txt data/imdb_registered2.txt .tok & 27 | perl to_tokens.pl data/imdb-test-neg.txt data/imdb_registered2.txt .tok & 28 | wait 29 | echo Done ... 30 | 31 | for set in train test; do 32 | for ext in .txt.tok .cat; do 33 | #--- merge the files for positive and negative. 34 | mv data/imdb-${set}-pos${ext} data/imdb-${set}${ext} 35 | cat data/imdb-${set}-neg${ext} >> data/imdb-${set}${ext} 36 | rm data/imdb-${set}-neg${ext} 37 | done 38 | done 39 | -------------------------------------------------------------------------------- /examples/other-sh/shcnn-3unsemb-rcv1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ##### 3 | ##### IMDB: Training using three types of unsupervised embedding 4 | ##### 5 | ##### Step 1. Generate input files. 6 | ##### Step 2. Training. 7 | ##### 8 | ##### NOTE: Unsupervised embedding files are downloaded if sdir=for-semi. 9 | ##### 10 | 11 | #-----------------# 12 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 13 | mem=2 # pre-allocate 2GB device memory 14 | source sh-common.sh 15 | #-----------------# 16 | nm=rcv1 17 | nm1=rcv1-1m 18 | sdir=for-semi # <= Where the unsupervised embedding files are. Downloaded if sdir=for-semi. 19 | # sdir=$outdir # <= Use the files generated by shcnn-{unsup|parsup|unsup3}-rcv1.sh 20 | ##### 21 | ##### WARNING: If your system uses Big Endian (Motorola convention), you cannot use the 22 | ##### downloaded files! They are in the little-endian format (Intel convention)! 23 | ##### 24 | 25 | dim=100 # dimensionality of unsupervised embeddings 26 | rcv1dir=rcv1_data 27 | 28 | options="LowerCase UTF8" 29 | txt_ext=.txt.tok 30 | 31 | z=4 # to avoid name conflict with other scripts 32 | 33 | #--- Step 0. Prepare unsupervised embedding files. Downloaded if sdir=for-semi. 34 | pch_sz=20 35 | s_fn0=${nm}-uns-p${pch_sz}.dim${dim}.epo10.ReLayer0 # generated by shcnn-unsup-rcv1.sh 36 | s_fn1=${nm}-parsup-p20p${pch_sz}.dim${dim}.epo10.ReLayer0 # generated by shcnn-parsup-rcv1.sh 37 | s_fn2=${nm}-unsx3-p${pch_sz}.dim${dim}.epo10.ReLayer0 # generated by shcnn-unsup3-rcv1.sh 38 | for fn in $s_fn0 $s_fn1 $s_fn2; do 39 | find_file $sdir $fn; if [ $? != 0 ]; then echo $shnm: find_file failed.; exit 1; fi 40 | done 41 | s_fn0=${sdir}/${s_fn0}; s_fn1=${sdir}/${s_fn1}; s_fn2=${sdir}/${s_fn2} 42 | 43 | #--- Step 1. Generate input files. 44 | xvoc1=${tmpdir}/${nm}${z}-trn.vocab 45 | $exe $gpu write_word_mapping layer0_fn=$s_fn0 layer_type=Weight+ word_map_fn=$xvoc1 # extract word mapping from the unsupervised embedding file. 46 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 47 | 48 | xvoc3=${tmpdir}/${nm}${z}-trn-123gram.vocab 49 | $exe $gpu write_word_mapping layer0_fn=$s_fn2 layer_type=Weight+ word_map_fn=$xvoc3 # extract word mapping from the unsupervised embedding file. 50 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 51 | 52 | for set in train test; do 53 | #--- dataset#0: (bow) 54 | rnm=${tmpdir}/${nm}${z}-${set}-p${pch_sz}bow 55 | $prep_exe gen_regions Bow VariableStride WritePositions \ 56 | region_fn_stem=$rnm input_fn=${rcv1dir}/rcv1-1m-${set} vocab_fn=$xvoc1 \ 57 | $options text_fn_ext=$txt_ext label_fn_ext=.lvl2 \ 58 | label_dic_fn=data/rcv1-lvl2.catdic \ 59 | patch_size=$pch_sz patch_stride=2 padding=$((pch_sz-1)) 60 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 61 | 62 | #--- dataset#1: (bag-of-1-3grams) 63 | pos_fn=${rnm}.pos # make regions at the same locations as above. 64 | rnm=${tmpdir}/${nm}${z}-${set}-p${pch_sz}x3bow 65 | $prep_exe gen_regions Bow input_pos_fn=$pos_fn \ 66 | region_fn_stem=$rnm input_fn=${rcv1dir}/rcv1-1m-${set} vocab_fn=$xvoc3 \ 67 | $options text_fn_ext=$txt_ext RegionOnly \ 68 | patch_size=$pch_sz 69 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 70 | done 71 | 72 | 73 | #--- Step 2. Training. 74 | gpumem=${gpu}:4 # pre-allocate 4GB GPU memory. 75 | 76 | mynm=shcnn-3unsemb-${nm}-dim${dim} 77 | logfn=${logdir}/${mynm}.log 78 | csvfn=${csvdir}/${mynm}.csv 79 | echo 80 | echo Supervised training using 3 types of unsupervised embedding to produce additional input. 81 | echo This takes a while. See $logfn and $csvfn for progress. 82 | nodes=1000 83 | num_pool=10 84 | $exe $gpumem train V2 \ 85 | trnname=${nm}${z}-train-p${pch_sz} tstname=${nm}${z}-test-p${pch_sz} data_dir=$tmpdir \ 86 | dsno0=bow dsno1=x3bow \ 87 | reg_L2=1e-4 step_size=0.25 \ 88 | loss=Square mini_batch_size=100 momentum=0.9 random_seed=1 \ 89 | datatype=sparse \ 90 | num_epochs=100 ss_scheduler=Few ss_decay=0.1 ss_decay_at=80 \ 91 | layers=2 num_sides=3 \ 92 | 0layer_type=WeightS+ 0nodes=$nodes 0activ_type=Rect \ 93 | 0pooling_type=Avg 0num_pooling=$num_pool 0resnorm_type=Text \ 94 | 1layer_type=Patch 1patch_size=$num_pool \ 95 | 0side0_layer_type=Weight+ 0side0_layer_fn=$s_fn0 0side0_dsno=0 \ 96 | 0side1_layer_type=Weight+ 0side1_layer_fn=$s_fn1 0side1_dsno=0 \ 97 | 0side2_layer_type=Weight+ 0side2_layer_fn=$s_fn2 0side2_dsno=1 \ 98 | evaluation_fn=$csvfn test_interval=25 > $logfn 99 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 100 | 101 | rm -f ${tmpdir}/${nm}${z}* 102 | -------------------------------------------------------------------------------- /examples/other-sh/shcnn-bow-all-rcv1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #### Input: token file (one article per line; tokens are delimited by white space) 3 | #### label file (one label per line) 4 | #### The input files are not included in the package due to copyright. 5 | 6 | #-----------------# 7 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 8 | mem=4 # pre-allocate 4GB device memory 9 | source sh-common.sh 10 | #-----------------# 11 | inpdir=rcv1_data # <= change this to where rcv1 data is. 12 | 13 | z=alsz 14 | 15 | for sz in r02k r03k r04k r05k r10k 1m 2m 3m; do 16 | #--- Step 1. Generate vocabulary 17 | echo Generaing vocabulary from training data ... 18 | 19 | max_num=30000 20 | vocab_fn=${tmpdir}/rcv1${z}-${sz}_trn-${max_num}.vocab 21 | options="LowerCase UTF8" 22 | 23 | $prep_exe gen_vocab input_fn=${inpdir}/rcv1-${sz}-train.txt.tok vocab_fn=$vocab_fn max_vocab_size=$max_num \ 24 | $options WriteCount RemoveNumbers stopword_fn=data/rcv1_stopword.txt 25 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 26 | 27 | #--- Step 2. Generate region files (${tmpdir}/*.xsmatbcvar) and target files (${tmpdir}/*.y) for training and testing CNN. 28 | # We generate region vectors of the convolution layer and write them to a file, instead of making them 29 | # on the fly during training/testing. 30 | echo 31 | echo Generating region files ... 32 | 33 | pch_sz=20 34 | 35 | for set in train test; do 36 | rnm=${tmpdir}/rcv1${z}-${sz}-${set}-p${pch_sz} 37 | 38 | $prep_exe gen_regions \ 39 | Bow VariableStride \ 40 | region_fn_stem=$rnm input_fn=${inpdir}/rcv1-${sz}-${set} vocab_fn=$vocab_fn \ 41 | $options text_fn_ext=.txt.tok label_fn_ext=.lvl2 \ 42 | label_dic_fn=data/rcv1-lvl2.catdic \ 43 | patch_size=$pch_sz patch_stride=2 padding=$((pch_sz-1)) 44 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 45 | done 46 | 47 | 48 | #--- Step 3. Training and test using GPU 49 | mynm=shcnn-bow-${sz}-rcv1 50 | log_fn=${logdir}/${mynm}.log 51 | csv_fn=${csvdir}/${mynm}.csv 52 | 53 | ss=0.5 54 | lam=1e-4 55 | if [ "$sz" = "r02k" ] || [ "$sz" = "r03k" ]; then 56 | lam=1e-3 57 | fi 58 | if [ "$sz" = "3m" ]; then 59 | ss=0.25 60 | fi 61 | 62 | echo 63 | echo Training CNN and testing ... 64 | echo This takes a while. See $log_fn and $perf_fn for progress. 65 | nodes=500 # number of feature maps (weight vectors). 66 | plnum=10 67 | $exe $gpu:$mem train \ 68 | data_dir=$tmpdir trnname=rcv1${z}-${sz}-train-p${pch_sz} tstname=rcv1${z}-${sz}-test-p${pch_sz} \ 69 | datatype=sparse \ 70 | loss=Square num_epochs=100 \ 71 | reg_L2=$lam momentum=0.9 mini_batch_size=100 random_seed=1 \ 72 | step_size=$ss ss_scheduler=Few ss_decay=0.1 ss_decay_at=80 \ 73 | layers=2 0layer_type=Weight+ 0nodes=$nodes 0activ_type=Rect \ 74 | 0pooling_type=Avg 0num_pooling=$plnum 0resnorm_type=Text \ 75 | 1layer_type=Patch 1patch_size=$plnum \ 76 | test_interval=25 evaluation_fn=$csv_fn \ 77 | > ${log_fn} 78 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 79 | 80 | rm -f ${tmpdir}/rcv1${z}-${sz}* 81 | done 82 | -------------------------------------------------------------------------------- /examples/other-sh/shcnn-bow-rcv1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #### Input: token file (one article per line; tokens are delimited by white space): *.txt.tok 3 | #### label file (one label per line): *.cat 4 | #### The input files are not included in the package due to copyright. 5 | 6 | #-----------------# 7 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 8 | mem=2 # pre-allocate 2GB device memory 9 | source sh-common.sh 10 | #-----------------# 11 | inpdir=rcv1_data # <= change this to where rcv1 data is. 12 | z=bo # to avoid name conflict with other scripts 13 | 14 | #--- Step 1. Generate vocabulary 15 | echo Generaing vocabulary from training data ... 16 | max_num=30000 17 | vocab_fn=${tmpdir}/rcv1${z}_trn-${max_num}.vocab 18 | options="LowerCase UTF8" 19 | 20 | $prep_exe gen_vocab input_fn=${inpdir}/rcv1-1m-train.txt.tok vocab_fn=$vocab_fn max_vocab_size=$max_num \ 21 | $options WriteCount RemoveNumbers stopword_fn=data/rcv1_stopword.txt 22 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 23 | 24 | #--- Step 2. Generate region files (${tmpdir}/*.xsmatbcvar), target files (${tmpdir}/*.y) 25 | #--- and word-mapping files (${tmpdir}/*.xtext). 26 | echo; echo Generating region files ... 27 | p=20 28 | for set in train test; do 29 | rnm=${tmpdir}/rcv1${z}_${set}-p${p} 30 | $prep_exe gen_regions Bow VariableStride region_fn_stem=$rnm \ 31 | input_fn=${inpdir}/rcv1-1m-${set} vocab_fn=$vocab_fn \ 32 | $options label_fn_ext=.lvl2 label_dic_fn=data/rcv1-lvl2.catdic \ 33 | patch_size=$p patch_stride=2 padding=$((p-1)) 34 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 35 | done 36 | 37 | 38 | #--- Step 3. Training and test using GPU 39 | mynm=shcnn-bow-rcv1 40 | log_fn=${logdir}/${mynm}.log; csv_fn=${csvdir}/${mynm}.csv 41 | echo; echo Training and testing ... ; echo This takes a while. See $log_fn and $csv_fn for progress. 42 | plnum=10 # number of pooling units. 43 | $exe $gpu:$mem train datatype=sparse \ 44 | data_dir=$tmpdir trnname=rcv1${z}_train-p${p} tstname=rcv1${z}_test-p${p} \ 45 | save_fn=${outdir}/${mynm}-mod save_interval=100 \ 46 | loss=Square num_epochs=100 \ 47 | reg_L2=1e-4 momentum=0.9 mini_batch_size=100 random_seed=1 \ 48 | step_size=0.5 ss_scheduler=Few ss_decay=0.1 ss_decay_at=80 \ 49 | layers=2 0layer_type=Weight+ 0nodes=1000 0activ_type=Rect \ 50 | 0pooling_type=Avg 0num_pooling=$plnum 0resnorm_type=Text \ 51 | 1layer_type=Patch 1patch_size=$plnum \ 52 | test_interval=25 evaluation_fn=$csv_fn > ${log_fn} 53 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 54 | 55 | rm -f ${tmpdir}/rcv1${z}* 56 | -------------------------------------------------------------------------------- /examples/other-sh/shcnn-seq2-3k-imdb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #### Input: token file (one review per line; tokens are delimited by white space) 3 | #### label file (one label per line) 4 | #### These input files were generated by prep_imdb.sh and included in the package. 5 | #### To find the order of the data points, see prep_imdb.sh and the files at lst/. 6 | 7 | #-----------------# 8 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 9 | source sh-common.sh 10 | #-----------------# 11 | 12 | nm=imdb 13 | z=s23k 14 | 15 | #--- Step 1. Generate vocabulary 16 | echo Generaing vocabulary from training data ... 17 | 18 | max_num=10000 19 | vocab_fn=${tmpdir}/${nm}${z}_trn-${max_num}.vocab 20 | options="LowerCase UTF8" 21 | 22 | $prep_exe gen_vocab input_fn=data/${nm}-train.txt.tok vocab_fn=$vocab_fn max_vocab_size=$max_num \ 23 | $options WriteCount 24 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 25 | 26 | #--- Step 2. Generate region files (${tmpdir}/*.xsmatbcvar) and target files (${tmpdir}/*.y). 27 | echo 28 | echo Generating region files with region size 2 and 3 ... 29 | for pch_sz in 2 3; do 30 | for set in train test; do 31 | rnm=${tmpdir}/${nm}${z}_${set}-10kp${pch_sz} 32 | $prep_exe gen_regions $options region_fn_stem=$rnm \ 33 | input_fn=data/${nm}-${set} vocab_fn=$vocab_fn label_dic_fn=data/${nm}_cat.dic \ 34 | patch_size=$pch_sz padding=$((pch_sz-1)) 35 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 36 | done 37 | done 38 | 39 | 40 | #--- Step 3. Training and test using GPU 41 | mynm=shcnn-seq2-3k-${nm} 42 | log_fn=${logdir}/${mynm}.log 43 | csv_fn=${csvdir}/${mynm}.csv 44 | echo 45 | echo Training CNN and testing ... 46 | echo This takes a while. See $log_fn and $perf_fn for progress. 47 | $exe $gpu train datatype=sparse \ 48 | data_dir=$tmpdir trnname=${nm}${z}_train- tstname=${nm}${z}_test- \ 49 | dsno0=10kp2 dsno1=10kp3 \ 50 | loss=Square num_epochs=100 \ 51 | reg_L2=0 top_reg_L2=1e-3 top_dropout=0.5 \ 52 | momentum=0.9 mini_batch_size=100 random_seed=1 \ 53 | step_size=0.25 ss_scheduler=Few ss_decay=0.1 ss_decay_at=80 \ 54 | layers=2 conn=0-top,1-top ConcatConn \ 55 | 0layer_type=Weight+ 0dsno=0 \ 56 | 1layer_type=Weight+ 1dsno=1 \ 57 | nodes=3000 activ_type=Rect pooling_type=Max num_pooling=1 resnorm_type=Text \ 58 | test_interval=25 evaluation_fn=$csv_fn > ${log_fn} 59 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 60 | 61 | rm -f ${tmpdir}/${nm}${z}* -------------------------------------------------------------------------------- /examples/other-sh/shcnn-seq2-all-elec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #### Input: token file (one review per line; tokens are delimited by white space): *.txt.tok 3 | #### label file (one label per line): *.cat 4 | #### These input files were generated by prep_elec.sh and included in the package. 5 | 6 | #-----------------# 7 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 8 | mem=4 # pre-allocate 4GB device memory. 9 | source sh-common.sh 10 | #-----------------# 11 | nm=elec 12 | dir=elec # <= change this to where the Elec dataset is. 13 | 14 | options="LowerCase UTF8" 15 | 16 | for sz in 05k 10k 50k 100k 200k; do 17 | #--- Step 1. Generate vocabulary 18 | echo Generaing vocabulary from training data ... $sz ... 19 | 20 | max_num=30000 21 | vocab_fn=${tmpdir}/${nm}-${sz}_trn-${max_num}.vocab 22 | 23 | $prep_exe gen_vocab input_fn=${dir}/${nm}-${sz}-train.txt.tok vocab_fn=$vocab_fn max_vocab_size=$max_num \ 24 | $options WriteCount 25 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 26 | 27 | #--- Step 2. Generate region files (${tmpdir}/*.xsmatbcvar) and target files (${tmpdir}/*.y) for training and testing CNN. 28 | # We generate region vectors of the convolution layer and write them to a file, instead of making them 29 | # on the fly during training/testing. 30 | echo 31 | echo Generating region files ... $sz ... 32 | for set in train test; do 33 | for pch_sz in 3 4; do 34 | rnm=${tmpdir}/${nm}-${sz}_${set}-p${pch_sz} 35 | inp_fn=${dir}/${nm}-${sz}-${set} 36 | if [ "$set" = "test" ]; then 37 | inp_fn=data/${nm}-${set} 38 | fi 39 | 40 | #--- NOTE: The parameters are the same as IMDB. 41 | $prep_exe gen_regions $options region_fn_stem=$rnm \ 42 | input_fn=$inp_fn vocab_fn=$vocab_fn label_dic_fn=data/${nm}_cat.dic \ 43 | patch_size=${pch_sz} padding=$((pch_sz-1)) 44 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 45 | done 46 | done 47 | 48 | 49 | #--- Step 3. Training and test using GPU 50 | mynm=shcnn-seq2-${nm}-${sz} 51 | log_fn=${logdir}/${mynm}.log 52 | csv_fn=${csvdir}/${mynm}.csv 53 | echo 54 | echo Training CNN and testing ... $sz ... 55 | echo This takes a while. See $log_fn and $perf_fn for progress. 56 | nodes=1000 57 | if [ "$sz" = "05k" ] || [ "$sz" = "10k" ]; then 58 | stepsize=0.5 59 | else 60 | stepsize=0.25 61 | fi 62 | 63 | $exe $gpu:$mem train trnname=${nm}-${sz}_train- tstname=${nm}-${sz}_test- \ 64 | datatype=sparse dsno0=p3 dsno1=p4 data_dir=$tmpdir \ 65 | loss=Square num_epochs=100 reg_L2=0 top_reg_L2=1e-3 top_dropout=0.5 \ 66 | momentum=0.9 mini_batch_size=100 random_seed=1 \ 67 | step_size=$stepsize ss_scheduler=Few ss_decay=0.1 ss_decay_at=80 \ 68 | layers=2 conn=0-top,1-top ConcatConn \ 69 | 0layer_type=Weight+ 0dsno=0 \ 70 | 1layer_type=Weight+ 1dsno=1 \ 71 | nodes=$nodes activ_type=Rect pooling_type=Max num_pooling=1 resnorm_type=Text \ 72 | test_interval=25 evaluation_fn=$csv_fn > ${log_fn} 73 | if [ $? != 0 ]; then echo $shnm: traning failed.; exit 1; fi 74 | 75 | rm ${tmpdir}/${nm}-${sz}* 76 | done 77 | -------------------------------------------------------------------------------- /examples/sample-multilab.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #---- Example of multi-label categorization. 3 | #---- Input: token files (data/s-multilab-{train|test}.txt.tok) 4 | #---- label files (data/s-multilab-{train|test}.cat) 5 | #---- label dictionary (data/s-multilab.catdic) 6 | 7 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 8 | source sh-common.sh 9 | 10 | nm=s-multilab 11 | 12 | #--- Step 1. Generate vocabulary 13 | echo Generaing vocabulary from training data ... 14 | vocab_fn=${tmpdir}/s-multilab.voc 15 | options=LowerCase 16 | 17 | $prep_exe gen_vocab input_fn=data/${nm}-train.txt.tok vocab_fn=$vocab_fn $options max_vocab_size=30000 18 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 19 | 20 | #--- Step 2. Generate input files for training and testing. 21 | echo; echo Generating region files ... 22 | p=3 # region size 23 | pnm=p${p} 24 | for set in train test; do 25 | rnm=${tmpdir}/${nm}-${set}-${pnm} 26 | $prep_exe gen_regions MultiLabel region_fn_stem=$rnm \ 27 | input_fn=data/${nm}-${set} vocab_fn=$vocab_fn $options label_dic_fn=data/${nm}.catdic \ 28 | patch_size=$p padding=$(((p-1)/2)) 29 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 30 | done 31 | 32 | #--- Step 3. Training and test using GPU 33 | echo; echo Training with multi-label data ... 34 | $exe $gpu train MultiLabel test_interval=5 random_seed=1 \ 35 | datatype=sparse trnname=${nm}-train-${pnm} tstname=${nm}-test-${pnm} data_dir=$tmpdir \ 36 | layers=1 0layer_type=Weight+ 0nodes=500 0activ_type=Rect 0pooling_type=Max 0num_pooling=1 \ 37 | loss=BinLogi2 mini_batch_size=100 momentum=0.9 reg_L2=1e-4 step_size=0.25 top_dropout=0.5 \ 38 | num_epochs=20 ss_scheduler=Few ss_decay=0.1 ss_decay_at=15 39 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 40 | 41 | -------------------------------------------------------------------------------- /examples/sample-save-predict.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #--- Example of saving a model and using 'predict' 3 | 4 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 5 | source sh-common.sh 6 | options=LowerCase # This must be shared by gen_vocab and gen_regions. 7 | 8 | #--- Step 1. Generate vocabulary 9 | echo Generaing a vocabulary file from training data ... 10 | vocab_fn=${tmpdir}/s.voc 11 | $prep_exe gen_vocab $options input_fn=data/s-train.txt.tok vocab_fn=$vocab_fn max_vocab_size=30000 12 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 13 | 14 | #--- Step 2. Generate input files for training and testing. 15 | echo; echo Generating region files ... 16 | p=3 # region size 17 | pnm=p${p} # used in filenames 18 | for set in train test; do 19 | rnm=${tmpdir}/s-${set}-${pnm} 20 | $prep_exe gen_regions $options region_fn_stem=$rnm \ 21 | input_fn=data/s-${set} vocab_fn=$vocab_fn label_dic_fn=data/s-cat.dic \ 22 | patch_size=$p padding=$(((p-1)/2)) 23 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 24 | done 25 | 26 | #--- Step 3. Training and testing using GPU 27 | logfile=${logdir}/sample-save.log 28 | echo; echo Training and saving the models 29 | epo=20 # number of epochs 30 | ss=0.25 # step-size (learning rate) 31 | save_fn=${outdir}/s-train-${pnm}.mod # model filenames are generated from this. 32 | $exe $gpu train save_interval=10 save_fn=$save_fn \ 33 | test_interval=5 random_seed=1 \ 34 | datatype=sparse trnname=s-train-${pnm} tstname=s-test-${pnm} data_dir=$tmpdir \ 35 | layers=1 0layer_type=Weight+ 0nodes=500 0activ_type=Rect 0pooling_type=Max 0num_pooling=1 \ 36 | loss=Square mini_batch_size=100 momentum=0.9 reg_L2=1e-4 step_size=$ss top_dropout=0.5 \ 37 | num_epochs=$epo ss_scheduler=Few ss_decay=0.1 ss_decay_at=$((epo*4/5)) 38 | if [ $? != 0 ]; then echo $shnm: train failed.; exit 1; fi 39 | 40 | #--- Step 4. Apply the saved model to test data 41 | mod_fn=${save_fn}.epo${epo}.ReNet # path to the model file saved after $epo epochs. 42 | #--- binary output (faster) 43 | $exe $gpu predict model_fn=$mod_fn \ 44 | prediction_fn=${outdir}/s-test-${pnm}.pred.bin \ 45 | datatype=sparse tstname=s-test-${pnm} data_dir=$tmpdir 46 | if [ $? != 0 ]; then echo $shnm: predict failed.; exit 1; fi 47 | 48 | #--- text output (slower) 49 | $exe $gpu predict model_fn=$mod_fn \ 50 | prediction_fn=${outdir}/s-test-${pnm}.pred.txt WriteText \ 51 | datatype=sparse tstname=s-test-${pnm} data_dir=$tmpdir 52 | if [ $? != 0 ]; then echo $shnm: predict failed.; exit 1; fi 53 | -------------------------------------------------------------------------------- /examples/sample-wordvec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ##### 3 | ##### Example: Using given word vectors as input to CNN, as is typically done. 4 | ##### Word vectors are in one word2vec-format binary file. 5 | ##### 6 | ##### NOTE: If word vectors require too much GPU memory (causing "out of memory"), 7 | ##### reduce the vocabulary ($xvoc below). 8 | 9 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 10 | source sh-common.sh 11 | 12 | #----- Pre-trained word vectors ----- 13 | options=LowerCase 14 | dim=10 # dimensionality of word vectors. 15 | wvparam="wordvec_bin_fn=data/s-dim10.wvbin" # word vector file in word2vec binary format (Little Endian) 16 | # wvparam="wordvec_txt_fn=data/s-dim10.wvtxt" # word vector file in word2vec text format 17 | #-------------------------------------- 18 | 19 | #--- write vocabulary (word mapping) of word vectors to a file. 20 | wm_fn=${tmpdir}/s-wordvec.wmap.txt 21 | $prep_exe write_wv_word_mapping $wvparam word_map_fn=$wm_fn # IgnoreDupWords 22 | if [ $? != 0 ]; then echo $shnm: write_wv_word_mapping failed.; exit 1; fi 23 | 24 | #--- Extract training data vocabulary. 25 | trn_voc=${tmpdir}/s-trn.vocab 26 | $prep_exe gen_vocab input_fn=data/s-train.txt.tok vocab_fn=$trn_voc $options WriteCount max_vocab_size=30000 27 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 28 | 29 | #--- Merge the word vector vocabulary and training data vocabulary. 30 | xvoc=${tmpdir}/s-wv-trn.vocab 31 | $prep_exe merge_vocab input_fns=${wm_fn}+${trn_voc} vocab_fn=$xvoc 32 | if [ $? != 0 ]; then echo $shnm: merge_vocab failed.; exit 1; fi 33 | 34 | #--- NOTE: If word vectors require too much GPU memory (indicated by "out of memory"), 35 | #--- remove some words from $xvoc (vocabulary). 36 | 37 | #--- Convert text to one-hot vectors and generate target files. 38 | for set in train test; do 39 | #--- sparse region vectors of size 1 = one-hot vectors 40 | rnm=${tmpdir}/s-${set}-p1 41 | $prep_exe gen_regions $options NoSkip region_fn_stem=$rnm \ 42 | input_fn=data/s-${set} vocab_fn=$xvoc label_dic_fn=data/s-cat.dic \ 43 | patch_size=1 44 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 45 | done 46 | 47 | #--- Convert word2vec word vector files to a weight file. 48 | w_fn=${tmpdir}/s-wv-trn.dmatc # weight file 49 | $prep_exe adapt_word_vectors $wvparam word_map_fn=$xvoc weight_fn=$w_fn rand_param=0.01 random_seed=1 50 | if [ $? != 0 ]; then echo $shnm: adapt_word_vectors failed.; exit 1; fi 51 | 52 | #--- Training. 53 | #--- Layer-0: word embedding layer 54 | #--- Layer-1&2: convolution layer of region size 3, stride 1, padding 2. 55 | #--- followed by max-pooling with one pooling region. 56 | 57 | opt=0Fixed # Don't fine-tune word vectors. Comment out this line to fine-tune word vectors. 58 | $exe $gpu train layers=3 $opt \ 59 | 0layer_type=Weight+ 0activ_type=None 0nodes=$dim 0weight_fn=$w_fn 0NoIntercept \ 60 | 1layer_type=Patch 1patch_size=3 1patch_stride=1 1padding=2 \ 61 | 2layer_type=Weight+ 2activ_type=Rect 2nodes=500 2pooling_type=Max 2num_pooling=1 \ 62 | datatype=sparse data_dir=$tmpdir trnname=s-train-p1 tstname=s-test-p1 \ 63 | reg_L2=1e-4 step_size=0.1 loss=Square mini_batch_size=100 momentum=0.9 random_seed=1 \ 64 | num_epochs=20 ss_scheduler=Few ss_decay=0.1 ss_decay_at=15 test_interval=5 65 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 66 | -------------------------------------------------------------------------------- /examples/sample.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | gpu=-1 # <= change this to, e.g., "gpu=2" to use a specific GPU. 4 | source sh-common.sh 5 | 6 | options=LowerCase # This must be shared by gen_vocab and gen_regions 7 | 8 | #--- Step 1. Generate vocabulary 9 | echo Generaing a vocabulary file from training data ... 10 | vocab_fn=${tmpdir}/s.voc 11 | $prep_exe gen_vocab $options input_fn=data/s-train.txt.tok vocab_fn=$vocab_fn max_vocab_size=30000 12 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 13 | 14 | #--- Step 2. Generate input files for training and testing. 15 | echo; echo Generating input files for training/testing ... 16 | p=3 # region size 17 | for set in train test; do 18 | rnm=${tmpdir}/s-${set}-p${p} 19 | $prep_exe gen_regions $options region_fn_stem=$rnm \ 20 | input_fn=data/s-${set} vocab_fn=$vocab_fn label_dic_fn=data/s-cat.dic \ 21 | patch_size=$p padding=$(((p-1)/2)) 22 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 23 | done 24 | 25 | #--- Step 3. Training and testing using GPU 26 | echo; echo Training and testing ... See ${csvdir}/sample.csv 27 | epo=20 # number of epochs 28 | ss=0.25 # step-size (learning rate) 29 | $exe $gpu train layers=1 0layer_type=Weight+ 0nodes=500 0activ_type=Rect \ 30 | 0pooling_type=Max 0num_pooling=1 top_dropout=0.5 \ 31 | loss=Square mini_batch_size=100 momentum=0.9 reg_L2=1e-4 step_size=$ss \ 32 | datatype=sparse trnname=s-train-p${p} tstname=s-test-p${p} data_dir=$tmpdir \ 33 | num_epochs=$epo ss_scheduler=Few ss_decay=0.1 ss_decay_at=$((epo*4/5)) \ 34 | test_interval=5 random_seed=1 evaluation_fn=${csvdir}/sample.csv 35 | if [ $? != 0 ]; then echo $shnm: train failed.; exit 1; fi 36 | -------------------------------------------------------------------------------- /examples/sh-common.sh: -------------------------------------------------------------------------------- 1 | #*** Global variables *** 2 | prep_exe=../bin/prepText 3 | exe=../bin/reNet 4 | 5 | tmpdir=temp 6 | csvdir=csv 7 | logdir=log 8 | outdir=out 9 | lddir=large-data 10 | if [ ! -e $tmpdir ]; then mkdir $tmpdir; fi 11 | if [ ! -e $csvdir ]; then mkdir $csvdir; fi 12 | if [ ! -e $logdir ]; then mkdir $logdir; fi 13 | if [ ! -e $outdir ]; then mkdir $outdir; fi 14 | 15 | shnm=$(basename $0) 16 | url=riejohnson.com/software/gz 17 | _cmd_=() # Global variable for passing commands to do_jobs. 18 | 19 | 20 | #*** Functions *** 21 | #--- Download a file 22 | find_file () { 23 | local fnm=file_download 24 | local _dir=$1 25 | local _fn=$2 26 | if [ "$_dir" = "" ]; then echo $fnm: _dir is missing; return 1; fi 27 | if [ "$_fn" = "" ]; then echo $fnm: _fn is missing; return 1; fi 28 | if [ -e "${_dir}/${_fn}" ]; then echo ${_dir}/${_fn} exists.; return 0; fi 29 | 30 | if [ "$_dir" != "for-semi" ]; then echo ${_dir}/${_fn} does not exist. Generate it first.; return 1; fi 31 | 32 | #--- If directory is "for-semi", try downloading the file. 33 | local _gzdir=gz 34 | 35 | if [ ! -e $_gzdir ]; then mkdir $_gzdir; fi 36 | local _fn=${_fn}.tar.gz 37 | rm -f ${_gzdir}/${_fn} # remove residue if any 38 | echo $fnm: Downloading $_fn ... 39 | wget --directory-prefix=${_gzdir} ${url}/${_fn} # download the file 40 | if [ $? != 0 ]; then echo wget ${url}/${_fn} failed.; return 1; fi 41 | tar -xvf ${_gzdir}/${_fn} 42 | if [ $? != 0 ]; then tar -xvf ${_gzdir}/${_fn} failed.; return 1; fi 43 | rm -f ${_gzdir}/${_fn} # clean up ... 44 | return 0 45 | } 46 | 47 | #--- Create and wait for background processes 48 | #--- Input: global variable _cmd_ (array) 49 | do_jobs () { 50 | local fnm=do_jobs 51 | #--- parameter check 52 | local _max_jobs=$1; local _num=$2; local _logfn=$3 53 | if [ "$_max_jobs" = "" ]; then echo $fnm: 1st arg is missing.; return 1; fi 54 | if [ "$_num" = "" ]; then echo $fnm: 2nd arg is missing.; return 1; fi 55 | if [ "$_logfn" = "" ]; then echo $fnm: 3rd arg is missing.; return 1; fi # used only if parallel 56 | local i; for (( i=0; i<_num; i++ )); do 57 | local cmd=${_cmd_[i]} 58 | if [ "$cmd" = "" ]; then echo $fnm: $i-th cmd is missing.; return 1; fi 59 | done 60 | 61 | #--- serial processing 62 | if [ $_max_jobs -le 1 ] || [ $_num -le 1 ]; then 63 | echo $fnm: serial ... 64 | local i; for (( i=0; i<_num; i++ )); do 65 | ${_cmd_[i]} # forward process 66 | if [ $? != 0 ]; then echo $fnm: Failed.; return 1; fi 67 | done 68 | #--- parallel processing 69 | else 70 | echo $fnm: parallel ... 71 | for (( i=0; i<_num; )); do 72 | #--- Create up to $_max_jobs background processes. 73 | local pid=(); local pnum=0 74 | local j; for (( j=0; j<_max_jobs; j++ )); do 75 | local cx=$((i+j)) 76 | if [ $cx -ge $_num ]; then break; fi 77 | 78 | local cmd=${_cmd_[$cx]} 79 | local mylog=${logdir}/${_logfn}.${cx} 80 | echo $fnm: Calling $cx-th command. See $mylog for progress. 81 | $cmd > $mylog & # background process 82 | if [ $? != 0 ]; then echo $fnm: Failed to invoke the $cx-th command.; return 1; fi 83 | 84 | pid[$pnum]=$!; pnum=$((pnum+1)) # save the process id. 85 | done 86 | 87 | #--- Wait for the background processes to complete. 88 | echo Waiting for $pnum jobs ... 89 | local _do_exit=0 90 | local j; for (( j=0; j $logfn 89 | if [ $? != 0 ]; then echo $shnm: traning failed.; exit 1; fi 90 | 91 | rm -f ${tmpdir}/${nm}${z}* -------------------------------------------------------------------------------- /examples/shcnn-5unsemb-imdb-elec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #--- Semi-supervised shallow CNN with 2 unsupervised LSTM embeddings 3 | #--- and 3 unsupervised CNN embeddings. 4 | 5 | #-----------------# 6 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 7 | mem=4 # pre-allocate 4GB device memory 8 | source sh-common.sh 9 | #-----------------# 10 | 11 | nm=imdb; mb=50; supepo=100 # (supepo=50 may be enough) 12 | # nm=elec; mb=100; supepo=50 # <= Uncomment this to train/test on Elec. 13 | 14 | ddir=data 15 | lstmdir=for-semi # <= Change this to where LSTM embeddings are. Downloaded if lstmdir=for-semi. 16 | cnndir=for-semi # <= Change this to where CNN embeddings are. Downloaded if cnndir=for-semi. 17 | ##### 18 | ##### WARNING: If your system uses Big Endian (Motorola convention), you cannot use the 19 | ##### downloaded files! They are in the little-endian format (Intel convention)! 20 | ##### 21 | 22 | options="LowerCase UTF8" 23 | txt_ext=.txt.tok 24 | 25 | catdic=${ddir}/${nm}_cat.dic 26 | z=3 # to avoid name conflict with other scripts 27 | 28 | dim=100 29 | unsepo=30 30 | 31 | #--- Download unsupervised embedding files if required. 32 | lay_fn0=${nm}-LstmF-dim${dim}.lay.epo${unsepo}.ReLayer0 33 | lay_fn1=${nm}-LstmB-dim${dim}.lay.epo${unsepo}.ReLayer0 34 | for fn in $lay_fn0 $lay_fn1; do find_file $lstmdir $fn; if [ $? != 0 ]; then echo $shnm: find_file failed.; exit 1; fi; done 35 | lay_fn0=${lstmdir}/${lay_fn0}; lay_fn1=${lstmdir}/${lay_fn1} 36 | 37 | lay_fn2=${nm}-uns-p5.dim100.epo10.ReLayer0 38 | lay_fn3=${nm}-unsx3-p5.dim100.epo10.ReLayer0 39 | lay_fn4=${nm}-parsup-p3p5.dim100.epo10.ReLayer0 40 | for fn in $lay_fn2 $lay_fn3 $lay_fn4; do find_file $cnndir $fn; if [ $? != 0 ]; then echo $shnm: find_file failed.; exit 1; fi; done 41 | lay_fn2=${cnndir}/${lay_fn2}; lay_fn3=${cnndir}/${lay_fn3}; lay_fn4=${cnndir}/${lay_fn4} 42 | 43 | #--- 44 | voc01=${tmpdir}/${nm}${z}-01.wmap 45 | $exe $gpu write_word_mapping layer_type=LstmF layer0_fn=$lay_fn0 word_map_fn=$voc01 46 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 47 | 48 | for set in train test; do 49 | #--- Generate region files for unsupervised LSTM embeddings 50 | rnm=${tmpdir}/${nm}${z}-${set}-p1 51 | $prep_exe gen_regions NoSkip $options region_fn_stem=$rnm \ 52 | input_fn=${ddir}/${nm}-${set} vocab_fn=$voc01 label_dic_fn=$catdic \ 53 | patch_size=1 54 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 55 | 56 | #--- Generate region files for layer-0 57 | p=5 58 | rnm=${tmpdir}/${nm}${z}-${set}-p${p} 59 | $prep_exe gen_regions NoSkip $options RegionOnly \ 60 | region_fn_stem=$rnm input_fn=${ddir}/${nm}-${set} vocab_fn=$voc01 \ 61 | patch_size=$p padding=$(((p-1)/2)) 62 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 63 | done 64 | 65 | #--- Generate region files for unsupervised CNN embeddings 66 | wm2=${tmpdir}/${nm}${z}-2.wmap 67 | $exe $gpu write_word_mapping layer0_fn=$lay_fn2 layer_type=Weight+ word_map_fn=$wm2 68 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 69 | 70 | wm3=${tmpdir}/${nm}${z}-3.wmap 71 | $exe $gpu write_word_mapping layer0_fn=$lay_fn3 layer_type=Weight+ word_map_fn=$wm3 72 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 73 | 74 | wm4=${tmpdir}/${nm}${z}-4.wmap 75 | $exe $gpu write_word_mapping layer0_fn=$lay_fn4 layer_type=Weight+ word_map_fn=$wm4 76 | if [ $? != 0 ]; then echo $shnm: write_word_mapping failed.; exit 1; fi 77 | 78 | for set in train test; do 79 | for no in 2 3 4; do 80 | p=5 # region size 81 | rnm=${tmpdir}/${nm}${z}-${set}-${no}-p${p}bow 82 | $prep_exe gen_regions $options NoSkip Bow RegionOnly \ 83 | region_fn_stem=$rnm input_fn=${ddir}/${nm}-${set} vocab_fn=${tmpdir}/${nm}${z}-${no}.wmap \ 84 | patch_size=$p padding=$(((p-1)/2)) 85 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 86 | done 87 | done 88 | 89 | #--- Training with labeled data and five unsupervised embeddings 90 | mynm=shcnn-5unsemb-${nm} 91 | logfn=${logdir}/${mynm}.log; csvfn=${csvdir}/${mynm}.csv 92 | echo; echo Training with labeled data and five unsupervised embeddings ... see $logfn and $csvfn 93 | $exe $gpu:$mem train \ 94 | num_sides=5 0side0_layer_type=LstmF 0side0_layer_fn=$lay_fn0 0side1_layer_type=LstmB 0side1_layer_fn=$lay_fn1 \ 95 | 0side2_layer_type=$ctyp 0side2_layer_fn=$lay_fn2 0side2_dsno=1 \ 96 | 0side3_layer_type=$ctyp 0side3_layer_fn=$lay_fn3 0side3_dsno=2 \ 97 | 0side4_layer_type=$ctyp 0side4_layer_fn=$lay_fn4 0side4_dsno=3 \ 98 | 0dsno=4 \ 99 | top_dropout=0.5 test_mini_batch_size=300 \ 100 | max_loss=5 inc=5000 trnname=${nm}${z}-train- tstname=${nm}${z}-test- data_dir=$tmpdir \ 101 | test_interval=25 reg_L2=1e-4 step_size=0.1 evaluation_fn=$csvfn \ 102 | layers=2 loss=Square mini_batch_size=$mb momentum=0.9 random_seed=1 \ 103 | datatype=sparse dsno0=p1 dsno1=2-p5bow dsno2=3-p5bow dsno3=4-p5bow dsno4=p5 \ 104 | num_epochs=$supepo ss_scheduler=Few ss_decay=0.1 ss_decay_at=$((supepo*4/5)) \ 105 | 0layer_type=WeightS+ 0nodes=1000 0activ_type=Rect \ 106 | 1layer_type=Pooling 1num_pooling=1 1pooling_type=Max 1resnorm_type=Text \ 107 | > $logfn 108 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 109 | 110 | rm -f ${tmpdir}/${nm}${z}* 111 | -------------------------------------------------------------------------------- /examples/shcnn-seq-imdb-elec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #### Input: token file (one review per line; tokens are delimited by white space): *.txt.tok 3 | #### label file (one label per line): *.cat 4 | #### These input files were generated by prep_imdb.sh and included in the package. 5 | #### To find the order of the data points, see prep_imdb.sh and the files at lst/. 6 | 7 | #-----------------# 8 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 9 | mem=2 # pre-allocate 2GB device memory 10 | source sh-common.sh 11 | #-----------------# 12 | nm=imdb 13 | # nm=elec # <= Uncomment this to train/test on Elec 14 | z=sq # To avoid filename conflict with other scripts. 15 | options="LowerCase UTF8" 16 | 17 | #--- Step 1. Generate vocabulary 18 | echo Generaing vocabulary from training data ... 19 | max_num=30000 # maximum vocabulary size. 20 | vocab_fn=${tmpdir}/${nm}${z}_trn-${max_num}.vocab 21 | $prep_exe gen_vocab input_fn=data/${nm}-train.txt.tok vocab_fn=$vocab_fn max_vocab_size=$max_num \ 22 | $options WriteCount 23 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 24 | 25 | #--- Step 2. Generate region files (${tmpdir}/*.xsmatbcvar), target files (${tmpdir}/*.y), 26 | #--- and word-mapping files (${tmpdir}/*.xtext). 27 | echo; echo Generating region files ... 28 | p=3 # region size 29 | for set in train test; do 30 | $prep_exe gen_regions $options region_fn_stem=${tmpdir}/${nm}${z}_${set}-p${p} \ 31 | input_fn=data/${nm}-${set} vocab_fn=$vocab_fn label_dic_fn=data/${nm}_cat.dic \ 32 | patch_size=$p padding=$((p-1)) 33 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 34 | done 35 | 36 | #--- Step 3. Training and test using GPU 37 | mynm=shcnn-seq-${nm} 38 | log_fn=${logdir}/${mynm}.log; csv_fn=${csvdir}/${mynm}.csv 39 | echo; echo Training CNN and testing ... ; echo This takes a while. See $log_fn and $csv_fn for progress. 40 | $exe $gpu:$mem train data_dir=$tmpdir datatype=sparse \ 41 | trnname=${nm}${z}_train-p${p} tstname=${nm}${z}_test-p${p} \ 42 | save_fn=${outdir}/${mynm}-mod save_interval=100 \ 43 | loss=Square num_epochs=100 \ 44 | reg_L2=1e-4 momentum=0.9 mini_batch_size=100 random_seed=1 \ 45 | step_size=0.05 ss_scheduler=Few ss_decay=0.1 ss_decay_at=80 \ 46 | layers=1 0layer_type=Weight+ 0nodes=1000 0activ_type=Rect \ 47 | 0pooling_type=Max 0num_pooling=1 0resnorm_type=Text \ 48 | top_dropout=0.5 test_interval=25 evaluation_fn=$csv_fn > ${log_fn} 49 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 50 | 51 | rm -f ${tmpdir}/${nm}${z}* 52 | -------------------------------------------------------------------------------- /examples/shcnn-seq2-bown-imdb-elec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #### Input: token file (one review per line; tokens are delimited by white space): *.txt.tok 3 | #### label file (one label per line): *.cat 4 | #### These input files were generated by prep_imdb.sh and included in the package. 5 | #### To find the order of the data points, see prep_imdb.sh and the files at lst/. 6 | 7 | #-----------------# 8 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 9 | mem=2 # pre-allocate 2GB device memory. 10 | source sh-common.sh 11 | #-----------------# 12 | nm=imdb; psz0=2; psz1=3; reg=1e-4 13 | # nm=elec; psz0=3; psz1=4; reg=0 # <= Uncomment this to train/test on Elec 14 | z=s2bn # To avoid filename conflict. 15 | x_ext=.xsmatcvar 16 | 17 | #--- Step 1. Generate vocabulary for NB weights 18 | echo Generaing uni-, bi-, and tri-gram vocabulary from training data to make NB-weights ... 19 | options="LowerCase UTF8" 20 | voc123=${tmpdir}/${nm}${z}_trn-123gram.vocab 21 | rm -f $voc123 22 | for nn in 1 2 3; do 23 | vocab_fn=${tmpdir}/${nm}${z}_trn-${nn}gram.vocab 24 | $prep_exe gen_vocab input_fn=data/${nm}-train.txt.tok vocab_fn=$vocab_fn \ 25 | $options WriteCount n=$nn 26 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 27 | cat $vocab_fn >> $voc123 28 | done 29 | 30 | #--- Step 2-1. Generate NB-weights 31 | echo Generating NB-weights ... 32 | $prep_exe gen_nbw $options nbw_fn=${tmpdir}/${nm}${z}.nbw3.dmat \ 33 | vocab_fn=$voc123 train_fn=data/${nm}-train label_dic_fn=data/${nm}_cat.dic 34 | if [ $? != 0 ]; then echo $shnm: gen_nbw failed.; exit 1; fi 35 | 36 | #--- Step 2-2. Generate NB-weighted bag-of-ngram files ... 37 | echo; echo Generating NB-weighted bag-of-ngram files ... 38 | for set in train test; do 39 | $prep_exe gen_nbwfeat $options vocab_fn=$voc123 input_fn=data/${nm}-${set} \ 40 | output_fn_stem=${tmpdir}/${nm}${z}_${set}-nbw3 x_ext=$x_ext \ 41 | label_dic_fn=data/${nm}_cat.dic nbw_fn=${tmpdir}/${nm}${z}.nbw3.dmat 42 | if [ $? != 0 ]; then echo $shnm: gen_nbwfeat failed.; exit 1; fi 43 | done 44 | 45 | #--- Step 3. Generate vocabulty for CNN 46 | echo; echo Generating vocabulary from training data for CNN ... 47 | max_num=30000 48 | vocab_fn=${tmpdir}/${nm}${z}_trn-1gram.${max_num}.vocab 49 | $prep_exe gen_vocab input_fn=data/${nm}-train.txt.tok vocab_fn=$vocab_fn max_vocab_size=$max_num \ 50 | $options WriteCount 51 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 52 | 53 | #--- Step 4. Generate region files (${tmpdir}/*.xsmatbcvar) and target files (${tmpdir}/*.y) for training and testing CNN. 54 | # We generate region vectors of the convolution layer and write them to a file, instead of making them 55 | # on the fly during training/testing. 56 | echo; echo Generating region files ... 57 | for pch_sz in $psz0 $psz1; do 58 | for set in train test; do 59 | rnm=${tmpdir}/${nm}${z}_${set}-p${pch_sz} 60 | $prep_exe gen_regions $options region_fn_stem=$rnm \ 61 | input_fn=data/${nm}-${set} vocab_fn=$vocab_fn label_dic_fn=data/${nm}_cat.dic \ 62 | x_ext=$x_ext patch_size=$pch_sz padding=$((pch_sz-1)) 63 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 64 | done 65 | done 66 | 67 | #--- Step 5. Training and test using GPU 68 | mynm=shcnn-seq2-bown-${nm} 69 | log_fn=${logdir}/${mynm}.log; csv_fn=${csvdir}/${mynm}.csv 70 | echo; echo Training and testing ... ; echo This takes a while. See $log_fn and $csv_fn for progress. 71 | nodes0=20; nodes1=1000; nodes2=1000 # number of feature maps (weight vectors) 72 | $exe $gpu:$mem train datatype=sparse data_dir=$tmpdir trnname=${nm}${z}_train- tstname=${nm}${z}_test- \ 73 | dsno0=nbw3 dsno1=p${psz0} dsno2=p${psz1} \ 74 | x_ext=$x_ext \ 75 | num_epochs=100 ss_scheduler=Few ss_decay=0.1 ss_decay_at=80 \ 76 | loss=Square reg_L2=$reg top_reg_L2=1e-4 step_size=0.25 top_dropout=0.5 \ 77 | momentum=0.9 mini_batch_size=100 random_seed=1 \ 78 | layers=3 conn=0-top,1-top,2-top ConcatConn \ 79 | 0layer_type=Weight+ 1layer_type=Weight+ 2layer_type=Weight+ \ 80 | 0nodes=$nodes0 0dsno=0 \ 81 | 1nodes=$nodes1 1dsno=1 \ 82 | 2nodes=$nodes2 2dsno=2 \ 83 | activ_type=Rect pooling_type=Max num_pooling=1 resnorm_type=Text \ 84 | test_interval=25 evaluation_fn=$csv_fn > ${log_fn} 85 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 86 | 87 | rm -f ${tmpdir}/${nm}${z}* 88 | -------------------------------------------------------------------------------- /examples/shcnn-seq2-imdb-elec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #### Input: token file (one review per line; tokens are delimited by white space) 3 | #### label file (one label per line) 4 | #### These input files were generated by prep_imdb.sh and included in the package. 5 | #### To find the order of the data points, see prep_imdb.sh and the files at lst/. 6 | 7 | #-----------------# 8 | gpu=-1 # <= change this to, e.g., "gpu=0" to use a specific GPU. 9 | source sh-common.sh 10 | #-----------------# 11 | nm=imdb; psz0=2; psz1=3 12 | # nm=elec; psz0=3; psz1=4 # <= Uncomment this to train/test on Elec. 13 | 14 | z=s2 # To avoid filename conflict. 15 | options="LowerCase UTF8" 16 | #--- Step 1. Generate vocabulary 17 | echo Generaing vocabulary from training data ... 18 | max_num=30000 19 | vocab_fn=${tmpdir}/${nm}${z}_trn-${max_num}.vocab 20 | $prep_exe gen_vocab input_fn=data/${nm}-train.txt.tok vocab_fn=$vocab_fn max_vocab_size=$max_num \ 21 | $options WriteCount 22 | if [ $? != 0 ]; then echo $shnm: gen_vocab failed.; exit 1; fi 23 | 24 | #--- Step 2. Generate region files (${tmpdir}/*.xsmatbcvar), target files (${tmpdir}/*.y), and 25 | #--- word-mapping files (${tmpdir}/*.xtext). 26 | echo; echo Generating region files with region size 2 and 3 ... 27 | for p in $psz0 $psz1; do # region sizes 28 | for set in train test; do 29 | $prep_exe gen_regions $options region_fn_stem=${tmpdir}/${nm}${z}_${set}-p${p} \ 30 | input_fn=data/${nm}-${set} vocab_fn=$vocab_fn label_dic_fn=data/${nm}_cat.dic \ 31 | patch_size=$p padding=$((p-1)) 32 | if [ $? != 0 ]; then echo $shnm: gen_regions failed.; exit 1; fi 33 | done 34 | done 35 | 36 | #--- Step 3. Training and test using GPU 37 | mynm=shcnn-seq2-${nm} 38 | log_fn=${logdir}/${mynm}.log; csv_fn=${csvdir}/${mynm}.csv 39 | echo; echo Training CNN and testing ... ; echo This takes a while. See $log_fn and $csv_fn for progress. 40 | $exe $gpu train datatype=sparse \ 41 | data_dir=$tmpdir trnname=${nm}${z}_train- tstname=${nm}${z}_test- \ 42 | dsno0=p${psz0} dsno1=p${psz1} \ 43 | loss=Square num_epochs=100 \ 44 | reg_L2=0 top_reg_L2=1e-3 top_dropout=0.5 \ 45 | momentum=0.9 mini_batch_size=100 random_seed=1 \ 46 | step_size=0.25 ss_scheduler=Few ss_decay=0.1 ss_decay_at=80 \ 47 | layers=2 conn=0-top,1-top ConcatConn \ 48 | 0layer_type=Weight+ 0dsno=0 \ 49 | 1layer_type=Weight+ 1dsno=1 \ 50 | nodes=1000 activ_type=Rect pooling_type=Max num_pooling=1 resnorm_type=Text \ 51 | test_interval=25 evaluation_fn=$csv_fn > ${log_fn} 52 | if [ $? != 0 ]; then echo $shnm: training failed.; exit 1; fi 53 | 54 | rm -f ${tmpdir}/${nm}${z}* 55 | -------------------------------------------------------------------------------- /imdb-data.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/riejohnson/ConText/4f51d8db5bffa8278237353f3a6bfa2ee9785071/imdb-data.tar.gz -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | #### Change CUDA_PATH and -gencode options in CFLAGS1 if necessary. 2 | #### 3 | #### This makefile works fine with CUDA 8.0 and Maxwell/Pascal GPUs. 4 | #### 5 | #### If your CUDA version is older, you may need to remove some "-gencode" 6 | #### line(s) that your CUDA does not support, to avoid compile error, 7 | #### or try makefile7.5, which works with CUDA 7.5. 8 | #### 9 | #### If the compute capability of your GPU card is older than 3.0, 10 | #### try makefile 7.5, which works with CUDA 7.5. 11 | #### 12 | #### If the compute capability of your GPU card is 7.0 or higher, 13 | #### see NVIDIA documentation. 14 | #### 15 | #### The compute capability can be found by looking up Wikipedia: 16 | #### https://en.wikipedia.org/wiki/CUDA . 17 | #### It can be also found by entering "gpuDevice" in matlab. 18 | #### 19 | ####-------------------------------------------------------------------------- 20 | #### NOTE: This makefile is for GitHub users only. In addition to compiling 21 | #### source code, it decompresses sample text files by "tar -xvf" and makes 22 | #### shell script files executable by "chmod +x". These additional actions 23 | #### are for going around GitHub restrictions. If they cause any trouble, 24 | #### please try the archive at riejohnson.com/cnn_download.html#download 25 | #### instead of GitHub. 26 | ####-------------------------------------------------------------------------- 27 | 28 | BIN_NAME1 = reNet 29 | BIN_DIR = bin 30 | TARGET1 = $(BIN_DIR)/$(BIN_NAME1) 31 | 32 | CUDA_PATH = /usr/local/cuda# <= Change this 33 | #CUDA_PATH = /opt/sw/packages/cuda/8.0# 34 | CUDA_INC_PATH = $(CUDA_PATH)/include 35 | CUDA_BIN_PATH = $(CUDA_PATH)/bin 36 | CUDA_LIB_PATH = $(CUDA_PATH)/lib64 37 | LDFLAGS1 = -L$(CUDA_LIB_PATH) -lcudart -lcublas -lcurand -lcusparse 38 | CFLAGS1 = -Isrc/com -Isrc/data -Isrc/nnet -D__AZ_SMAT_SINGLE__ -D__AZ_GPU__ -I$(CUDA_INC_PATH) -O2 \ 39 | -gencode arch=compute_30,code=sm_30 \ 40 | -gencode arch=compute_32,code=sm_32 \ 41 | -gencode arch=compute_35,code=sm_35 \ 42 | -gencode arch=compute_37,code=sm_37 \ 43 | -gencode arch=compute_50,code=sm_50 \ 44 | -gencode arch=compute_52,code=sm_52 \ 45 | -gencode arch=compute_53,code=sm_53 \ 46 | -gencode arch=compute_60,code=sm_60 \ 47 | -gencode arch=compute_61,code=sm_61 \ 48 | -gencode arch=compute_62,code=sm_62 49 | 50 | CPP_FILES1= \ 51 | src/com/AzDmat.cpp \ 52 | src/com/AzParam.cpp \ 53 | src/com/AzSmat.cpp \ 54 | src/com/AzStrPool.cpp \ 55 | src/com/AzTextMat.cpp \ 56 | src/com/AzTools.cpp \ 57 | src/com/AzUtil.cpp \ 58 | src/nnet/AzpPatchDflt.cpp \ 59 | src/nnet/AzpReNet.cpp \ 60 | src/nnet/AzpReLayer.cpp \ 61 | src/nnet/AzMultiConn.cpp \ 62 | src/nnet/AzpMain_reNet.cpp \ 63 | src/nnet/AzpLmSgd.cpp \ 64 | src/nnet/AzPmat.cpp \ 65 | src/nnet/AzPmatSpa.cpp \ 66 | src/nnet/AzPmatApp.cpp \ 67 | src/nnet/AzPmat_gpu.cu \ 68 | src/nnet/AzPmatSpa_gpu.cu \ 69 | src/nnet/AzCuda_Pmat.cu \ 70 | src/nnet/AzCuda_PmatSpa.cu \ 71 | src/nnet/AzCuda_PmatApp.cu \ 72 | src/nnet/AzpEv.cpp \ 73 | src/nnet/AzpLossDflt.cpp \ 74 | src/nnet/driv_reNet.cpp 75 | 76 | 77 | BIN_NAME2 = prepText 78 | TARGET2 = $(BIN_DIR)/$(BIN_NAME2) 79 | CFLAGS2 = -Isrc/com -O2 -D__AZ_SMAT_SINGLE__ 80 | 81 | CPP_FILES2= \ 82 | src/com/AzDmat.cpp \ 83 | src/com/AzParam.cpp \ 84 | src/com/AzSmat.cpp \ 85 | src/com/AzStrPool.cpp \ 86 | src/com/AzTools.cpp \ 87 | src/com/AzTextMat.cpp \ 88 | src/com/AzUtil.cpp \ 89 | src/data/AzPrepText.cpp \ 90 | src/data/AzTools_text.cpp \ 91 | src/data/driv_PrepText.cpp 92 | 93 | 94 | TARGET0 = examples/data/imdb-train.txt.tok 95 | 96 | all: $(TARGET1) $(TARGET2) $(TARGET0) 97 | 98 | ${TARGET0}: 99 | tar -xvf imdb-data.tar.gz 100 | chmod +x examples/*.sh 101 | chmod +x examples/other-sh/*.sh 102 | 103 | ${TARGET1}: 104 | mkdir -p bin 105 | /bin/rm -f $(TARGET1) 106 | $(CUDA_BIN_PATH)/nvcc $(CPP_FILES1) $(CFLAGS1) -o $(TARGET1) $(LDFLAGS1) 107 | 108 | ${TARGET2}: 109 | /bin/rm -f $(TARGET2) 110 | g++ $(CPP_FILES2) $(CFLAGS2) -o $(TARGET2) 111 | 112 | clean: 113 | /bin/rm -f $(TARGET1) 114 | /bin/rm -f $(TARGET2) 115 | 116 | cleandata: 117 | /bin/rm -f $(TARGET0) -------------------------------------------------------------------------------- /makefile7.5: -------------------------------------------------------------------------------- 1 | #### Change CUDA_PATH and -gencode options in CFLAGS1 if necessary. 2 | #### 3 | #### This makefile works fine with CUDA 7.5 and Kepler/Maxwell GPUs. 4 | #### 5 | #### If your CUDA version is older, you may need to remove some "-gencode" 6 | #### line(s) that your CUDA does not support, to avoid compile error. 7 | #### 8 | #### If the compute capability of your GPU card is 6.0 or higher, 9 | #### you need to add a "-gencode" line for it, or try "makefile". 10 | #### 11 | #### The compute capability can be found by looking up Wikipedia: 12 | #### https://en.wikipedia.org/wiki/CUDA . 13 | #### It can be also found by entering "gpuDevice" in matlab. 14 | #### 15 | ####-------------------------------------------------------------------------- 16 | #### NOTE: This makefile is for GitHub users only. In addition to compiling 17 | #### source code, it decompresses sample text files by "tar -xvf" and makes 18 | #### shell script files executable by "chmod +x". These additional actions 19 | #### are for going around GitHub restrictions. If they cause any trouble, 20 | #### please try the archive at riejohnson.com/cnn_download.html#download 21 | #### instead of GitHub. 22 | ####-------------------------------------------------------------------------- 23 | 24 | BIN_NAME1 = reNet 25 | BIN_DIR = bin 26 | TARGET1 = $(BIN_DIR)/$(BIN_NAME1) 27 | 28 | CUDA_PATH = /usr/local/cuda# <= Change this 29 | #CUDA_PATH = /opt/sw/packages/cuda/7.5# 30 | CUDA_INC_PATH = $(CUDA_PATH)/include 31 | CUDA_BIN_PATH = $(CUDA_PATH)/bin 32 | CUDA_LIB_PATH = $(CUDA_PATH)/lib64 33 | LDFLAGS1 = -L$(CUDA_LIB_PATH) -lcudart -lcublas -lcurand -lcusparse 34 | CFLAGS1 = -Isrc/com -Isrc/data -Isrc/nnet -D__AZ_SMAT_SINGLE__ -D__AZ_GPU__ -I$(CUDA_INC_PATH) -O2 \ 35 | -gencode arch=compute_20,code=sm_20 \ 36 | -gencode arch=compute_20,code=sm_21 \ 37 | -gencode arch=compute_30,code=sm_30 \ 38 | -gencode arch=compute_32,code=sm_32 \ 39 | -gencode arch=compute_35,code=sm_35 \ 40 | -gencode arch=compute_37,code=sm_37 \ 41 | -gencode arch=compute_50,code=sm_50 \ 42 | -gencode arch=compute_52,code=sm_52 \ 43 | -gencode arch=compute_53,code=sm_53 44 | 45 | CPP_FILES1= \ 46 | src/com/AzDmat.cpp \ 47 | src/com/AzParam.cpp \ 48 | src/com/AzSmat.cpp \ 49 | src/com/AzStrPool.cpp \ 50 | src/com/AzTextMat.cpp \ 51 | src/com/AzTools.cpp \ 52 | src/com/AzUtil.cpp \ 53 | src/nnet/AzpPatchDflt.cpp \ 54 | src/nnet/AzpReNet.cpp \ 55 | src/nnet/AzpReLayer.cpp \ 56 | src/nnet/AzMultiConn.cpp \ 57 | src/nnet/AzpMain_reNet.cpp \ 58 | src/nnet/AzpLmSgd.cpp \ 59 | src/nnet/AzPmat.cpp \ 60 | src/nnet/AzPmatSpa.cpp \ 61 | src/nnet/AzPmatApp.cpp \ 62 | src/nnet/AzPmat_gpu.cu \ 63 | src/nnet/AzPmatSpa_gpu.cu \ 64 | src/nnet/AzCuda_Pmat.cu \ 65 | src/nnet/AzCuda_PmatSpa.cu \ 66 | src/nnet/AzCuda_PmatApp.cu \ 67 | src/nnet/AzpEv.cpp \ 68 | src/nnet/AzpLossDflt.cpp \ 69 | src/nnet/driv_reNet.cpp 70 | 71 | 72 | BIN_NAME2 = prepText 73 | TARGET2 = $(BIN_DIR)/$(BIN_NAME2) 74 | CFLAGS2 = -Isrc/com -O2 -D__AZ_SMAT_SINGLE__ 75 | 76 | CPP_FILES2= \ 77 | src/com/AzDmat.cpp \ 78 | src/com/AzParam.cpp \ 79 | src/com/AzSmat.cpp \ 80 | src/com/AzStrPool.cpp \ 81 | src/com/AzTools.cpp \ 82 | src/com/AzTextMat.cpp \ 83 | src/com/AzUtil.cpp \ 84 | src/data/AzPrepText.cpp \ 85 | src/data/AzTools_text.cpp \ 86 | src/data/driv_PrepText.cpp 87 | 88 | 89 | TARGET0 = examples/data/imdb-train.txt.tok 90 | 91 | all: $(TARGET1) $(TARGET2) $(TARGET0) 92 | 93 | ${TARGET0}: 94 | tar -xvf imdb-data.tar.gz 95 | chmod +x examples/*.sh 96 | chmod +x examples/other-sh/*.sh 97 | 98 | ${TARGET1}: 99 | mkdir -p bin 100 | /bin/rm -f $(TARGET1) 101 | $(CUDA_BIN_PATH)/nvcc $(CPP_FILES1) $(CFLAGS1) -o $(TARGET1) $(LDFLAGS1) 102 | 103 | ${TARGET2}: 104 | /bin/rm -f $(TARGET2) 105 | g++ $(CPP_FILES2) $(CFLAGS2) -o $(TARGET2) 106 | 107 | clean: 108 | /bin/rm -f $(TARGET1) 109 | /bin/rm -f $(TARGET2) 110 | 111 | cleandata: 112 | /bin/rm -f $(TARGET0) 113 | -------------------------------------------------------------------------------- /src/com/AzOut.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzOut.hpp 3 | * Copyright (C) 2011, 2012 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZ_OUT_HPP_ 20 | #define _AZ_OUT_HPP_ 21 | 22 | class AzOut { 23 | protected: 24 | bool isActive; 25 | int level; 26 | public: 27 | ostream *o; 28 | 29 | inline AzOut() : o(NULL), isActive(true), level(0) {} 30 | inline AzOut(ostream *o_ptr) : isActive(true), level(0) { 31 | o = o_ptr; 32 | } 33 | inline void reset(ostream *o_ptr) { 34 | o = o_ptr; 35 | activate(); 36 | } 37 | 38 | inline void deactivate() { 39 | isActive = false; 40 | } 41 | inline void activate() { 42 | isActive = true; 43 | } 44 | inline void setStdout() { 45 | o = &cout; 46 | activate(); 47 | } 48 | inline void setStderr() { 49 | o = &cerr; 50 | activate(); 51 | } 52 | inline bool isNull() const { 53 | if (!isActive) return true; 54 | if (o == NULL) return true; 55 | return false; 56 | } 57 | inline void flush() const { 58 | if (o != NULL) o->flush(); 59 | } 60 | inline void setLevel(int inp_level) { 61 | level = inp_level; 62 | } 63 | inline int getLevel() const { 64 | return level; 65 | } 66 | }; 67 | #endif 68 | -------------------------------------------------------------------------------- /src/com/AzRandGen.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpRandGen.hpp 3 | * Copyright (C) 2014-2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZ_RAND_GEN_HPP_ 20 | #define _AZ_RAND_GEN_HPP_ 21 | 22 | #include "AzDmat.hpp" 23 | 24 | /* random number generator */ 25 | class AzRandGen { 26 | protected: 27 | bool do_myown; /* since 4/27/2017 */ 28 | int myown_seed; 29 | unsigned long long myown_val; 30 | public: 31 | AzRandGen() : do_myown(false), myown_seed(1), myown_val((unsigned long long)myown_seed) {} 32 | void _srand_(int seed) { 33 | do_myown = true; 34 | myown_seed = seed; 35 | myown_val = (unsigned long long)seed; 36 | } 37 | inline unsigned long long _rand_ULL() { 38 | if (!do_myown) return rand(); 39 | else { 40 | myown_val = myown_val * (unsigned long long)25214903917 + 11; 41 | return myown_val; 42 | } 43 | } 44 | int _rand_() { 45 | if (do_myown) return _rand_ULL() % (unsigned long long)2147483648; 46 | else return rand(); 47 | } 48 | 49 | /*--- Gaussian ---*/ 50 | void gaussian(double param, AzDmat *m) { /* must be formatted by caller */ 51 | AzX::throw_if((m == NULL), "AzRandGen::gaussian", "null pointer"); 52 | for (int col = 0; col < m->colNum(); ++col) gaussian(param, m->col_u(col)); 53 | } 54 | void gaussian(double param, AzDvect *v) { 55 | AzX::throw_if((v == NULL), "AzRandGen::gaussian", "null pointer"); 56 | gaussian(v->point_u(), v->rowNum()); 57 | v->multiply(param); 58 | } 59 | 60 | /*--- uniform ---*/ 61 | void uniform(double param, AzDmat *m) { /* must be formatted by caller */ 62 | for (int col = 0; col < m->colNum(); ++col) uniform(param, m->col_u(col)); 63 | } 64 | void uniform(double param, AzDvect *v) { 65 | AzX::throw_if((v == NULL), "AzRandGen::uniform", "null pointer"); 66 | double *val = v->point_u(); 67 | uniform_01(val, v->rowNum()); /* [0,1] */ 68 | for (int ex = 0; ex < v->rowNum(); ++ex) { 69 | val[ex] = (val[ex]*2-1)*param; /* -> [-param, param] */ 70 | } 71 | } 72 | void uniform_01(AzDvect &v) { uniform_01(v.point_u(), v.rowNum()); } 73 | void uniform_01(AzDmat &m) { for (int col = 0; col < m.colNum(); ++col) uniform_01(*m.col_u(col)); } 74 | template /* T: float | double */ 75 | void uniform_01(T *val, size_t sz) { /* [0,1] */ 76 | int denomi = 30000; 77 | for (size_t ix = 0; ix < sz; ++ix) { 78 | val[ix] = (T)((double)(_rand_() % denomi) / (double)denomi); 79 | } 80 | } 81 | 82 | /*--- polar method ---*/ 83 | template /* T: float | double */ 84 | void gaussian(T *val, size_t sz) { 85 | for (size_t ix = 0; ix < sz; ) { 86 | double u1 = 0, u2 = 0, s = 0; 87 | for ( ; ; ) { 88 | u1 = uniform1(); u2 = uniform1(); 89 | s = u1*u1 + u2*u2; 90 | if (s > 0 && s < 1) break; 91 | } 92 | double x1 = u1 * sqrt(-2*log(s)/s); 93 | val[ix++] = x1; 94 | if (ix >= sz) break; 95 | 96 | double x2 = u2 * sqrt(-2*log(s)/s); 97 | val[ix++] = x2; 98 | } 99 | } 100 | 101 | protected: 102 | virtual double uniform1() { 103 | int mymax = 32000; 104 | double mymaxmax = mymax * (mymax-1) + mymax-1; 105 | int rand0 = _rand_() % mymax; 106 | int rand1 = _rand_() % mymax; 107 | double output = (rand0 * mymax + rand1) / mymaxmax; /* (0,1) */ 108 | output = 2 * output - 1; /* (-1,1) */ 109 | return output; 110 | } 111 | }; 112 | 113 | #endif -------------------------------------------------------------------------------- /src/com/AzReadOnlyMatrix.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzReadOnlyMatrix.hpp 3 | * Copyright (C) 2011,2012,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZ_READONLY_MATRIX_HPP_ 20 | #define _AZ_READONLY_MATRIX_HPP_ 21 | 22 | #include "AzUtil.hpp" 23 | 24 | //! Abstract class: interface for read-only vectors. 25 | class AzReadOnlyVector { 26 | public: 27 | virtual int rowNum() const = 0; 28 | virtual double get(int row_no) const = 0; 29 | virtual int next(AzCursor &cursor, double &out_val) const = 0; 30 | virtual int nonZeroRowNum() const = 0; 31 | 32 | /*--------------------------------------*/ 33 | virtual void writeText(const char *fn, int digits) const { 34 | AzIntArr ia; 35 | ia.range(0, rowNum()); 36 | writeText(fn, &ia, digits); 37 | } 38 | 39 | /*--------------------------------------*/ 40 | virtual void writeText(const char *fn, const AzIntArr &ia, int digits) const { 41 | AzFile file(fn); file.open("wb"); 42 | AzBytArr s; 43 | for (int ix = 0; ix < ia.size(); ++ix) { 44 | double val = get(ia[ix]); 45 | s.cn(val, digits); s.nl(); 46 | } 47 | s.writeText(&file); 48 | file.close(true); 49 | } 50 | 51 | /*--------------------------------------*/ 52 | virtual void to_sparse(AzBytArr &s, int digits) const { 53 | AzCursor cur; 54 | for ( ; ; ) { 55 | double val; 56 | int row = next(cur, val); 57 | if (row < 0) break; 58 | if (s.length() > 0) s << ' '; 59 | s << row; 60 | if (val != 1) { 61 | s << ':'; s.cn(val, digits); 62 | } 63 | } 64 | } 65 | 66 | /*--------------------------------------*/ 67 | virtual void to_dense(AzBytArr &s, int digits) const { 68 | for (int row = 0; row < rowNum(); ++row) { 69 | if (row > 0) s << " "; 70 | s.cn(get(row), digits); 71 | } 72 | } 73 | }; 74 | 75 | //! Abstract class: interface for read-only matrices. 76 | class AzReadOnlyMatrix { 77 | public: 78 | virtual int rowNum() const = 0; 79 | virtual int colNum() const = 0; 80 | virtual double get(int row_no, int col_no) const = 0; 81 | virtual const AzReadOnlyVector *col(int col_no) const = 0; 82 | 83 | /*--------------------------------------*/ 84 | virtual void writeText(const char *fn, int digits, 85 | bool doSparse=false, 86 | bool doAppend=false) const { 87 | AzIntArr ia; ia.range(0, colNum()); 88 | writeText(fn, ia, digits, doSparse, doAppend); 89 | } 90 | 91 | /*--------------------------------------*/ 92 | virtual void writeText(const char *fn, const AzIntArr &ia, int digits, 93 | bool doSparse=false, 94 | bool doAppend=false) const { 95 | if (doSparse) { 96 | AzX::throw_if(doAppend, "AzReadOnlyMatrix::writeText", "sparse matrices cannot be appended"); 97 | writeText_sparse(fn, ia, digits); 98 | return; 99 | } 100 | AzFile file(fn); 101 | if (doAppend) file.open("ab"); 102 | else file.open("wb"); 103 | for (int ix = 0; ix < ia.size(); ++ix) { 104 | int cx = ia[ix]; 105 | AzBytArr s; col(cx)->to_dense(s, digits); s.nl(); 106 | s.writeText(&file); 107 | } 108 | file.close(true); 109 | } 110 | 111 | /*--------------------------------------*/ 112 | virtual void writeText_sparse(const char *fn, const AzIntArr &ia, int digits) const { 113 | AzFile file(fn); 114 | file.open("wb"); 115 | AzBytArr s_header("sparse "); s_header.cn(rowNum()); s_header.nl(); 116 | s_header.writeText(&file); 117 | for (int ix = 0; ix < ia.size(); ++ix) { 118 | int cx = ia[ix]; 119 | AzBytArr s; col(cx)->to_sparse(s, digits); s.nl(); 120 | s.writeText(&file); 121 | } 122 | file.close(true); 123 | } 124 | }; 125 | #endif 126 | -------------------------------------------------------------------------------- /src/data/driv_PrepText.cpp: -------------------------------------------------------------------------------- 1 | #define _AZ_MAIN_ 2 | #include "AzUtil.hpp" 3 | #include "AzPrepText.hpp" 4 | 5 | void help() { 6 | cout << "action: gen_vocab | gen_regions | gen_regions_unsup | gen_regions_parsup | merge_vocab | split_text | adapt_word_vectors | gen_nbw | gen_nbwfeat | gen_b_feat" << endl; 7 | cout << endl; 8 | cout << "Enter, for example, \"prepText gen_vocab\" to print help for a specific action." << endl; 9 | } 10 | 11 | /*******************************************************************/ 12 | /* main */ 13 | /*******************************************************************/ 14 | int main(int argc, const char *argv[]) { 15 | AzException *stat = NULL; 16 | 17 | if (argc < 2) { 18 | help(); 19 | return -1; 20 | } 21 | 22 | const char *action = argv[1]; 23 | int oo = 1; /* pass action too */ 24 | 25 | try { 26 | Az_check_system2_(); 27 | AzPrepText prep(log_out); 28 | if (strcmp(action, "gen_vocab") == 0) prep.gen_vocab(argc-oo, argv+oo); 29 | else if (strcmp(action, "gen_regions") == 0) prep.gen_regions(argc-oo, argv+oo); 30 | else if (strcmp(action, "show_regions") == 0) prep.show_regions(argc-oo, argv+oo); 31 | else if (strcmp(action, "gen_nbw") == 0) prep.gen_nbw(argc-oo, argv+oo); 32 | else if (strcmp(action, "gen_nbwfeat") == 0) prep.gen_nbwfeat(argc-oo, argv+oo); 33 | else if (strcmp(action, "gen_b_feat") == 0) prep.gen_b_feat(argc-oo, argv+oo); 34 | else if (strcmp(action, "split_text") == 0) prep.split_text(argc-oo, argv+oo); 35 | else if (strcmp(action, "gen_regions_unsup") == 0) prep.gen_regions_unsup(argc-oo, argv+oo); 36 | else if (strcmp(action, "gen_regions_parsup") == 0) prep.gen_regions_parsup(argc-oo, argv+oo); 37 | else if (strcmp(action, "show_regions_XY") == 0) prep.show_regions_XY(argc-oo, argv+oo); 38 | else if (strcmp(action, "merge_vocab") == 0) prep.merge_vocab(argc-oo, argv+oo); 39 | else if (strcmp(action, "adapt_word_vectors") == 0) prep.adapt_word_vectors(argc-oo, argv+oo); 40 | else if (strcmp(action, "write_wv_word_mapping") == 0) prep.write_wv_word_mapping(argc-oo, argv+oo); 41 | else { 42 | help(); 43 | return -1; 44 | } 45 | } 46 | catch (AzException *e) { 47 | stat = e; 48 | } 49 | 50 | if (stat != NULL) { 51 | cout << stat->getMessage() << endl; 52 | return -1; 53 | } 54 | 55 | return 0; 56 | } 57 | 58 | -------------------------------------------------------------------------------- /src/nnet/Az2D.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * Azp2D.hpp 3 | * Copyright (C) 2014-2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZ_2D_HPP_ 20 | #define _AZ_2D_HPP_ 21 | 22 | #include "AzUtil.hpp" 23 | #include "AzPrint.hpp" 24 | #include "AzxD.hpp" 25 | 26 | class Az2D { /* rectangle */ 27 | public: 28 | int x0, xsz, y0, ysz; 29 | int x1, y1; 30 | 31 | void copy_from(const AzxD *xd) { 32 | if (xd->get_dim() == 1) reset(xd->sz(0), 1); 33 | else if (xd->get_dim() == 2) reset(xd->sz(0), xd->sz(1)); 34 | else AzX::throw_if(true, "Az2D::reset(AzxD)", "dim mismatch"); 35 | } 36 | void copy_to(AzxD *xd) const { 37 | AzX::throw_if((x0 != 0 || y0 != 0), "Az2D::copy_to(xd)", "must be located at (0,0)"); 38 | int arr[2] = {xsz,ysz}; 39 | xd->reset(arr, 2); 40 | } 41 | 42 | /*--- ---*/ 43 | inline bool isSame(const Az2D &inp) const { 44 | return isSame(&inp); 45 | } 46 | inline bool isSame(const Az2D *inp) const { 47 | if (x0 != inp->x0 || y0 != inp->y0 || xsz != inp->xsz || ysz != inp->ysz) return false; 48 | return true; 49 | } 50 | Az2D() : x0(0), xsz(0), y0(0), ysz(0), x1(0), y1(0) {} 51 | Az2D(int sz0, int sz1) { 52 | reset(sz0, sz1); 53 | } 54 | Az2D(const AzxD *xd) { 55 | copy_from(xd); 56 | } 57 | void reset(int sz0, int sz1) { 58 | x0 = 0; xsz = sz0; y0 = 0; ysz = sz1; 59 | sync(); 60 | } 61 | inline int size() const { 62 | return xsz*ysz; 63 | } 64 | void reset(int inp_x0, int inp_xsz, int inp_y0, int inp_ysz) { 65 | x0=inp_x0; xsz=inp_xsz; y0=inp_y0; ysz=inp_ysz; 66 | sync(); 67 | } 68 | void write(AzFile *file) const { 69 | file->writeInt(x0); 70 | file->writeInt(xsz); 71 | file->writeInt(y0); 72 | file->writeInt(ysz); 73 | } 74 | void read(AzFile *file) { 75 | x0 = file->readInt(); 76 | xsz = file->readInt(); 77 | y0 = file->readInt(); 78 | ysz = file->readInt(); 79 | sync(); 80 | } 81 | void show(const char *header, AzOut &out) const { 82 | AzBytArr s(header); s.c("("); s.cn(x0); s.c(","); s.cn(y0); s.c(") - ("); 83 | s.cn(x1); s.c(","); s.cn(y1); s.c(")"); 84 | AzPrint::writeln(out, s); 85 | } 86 | 87 | /*------------------------------------------------------------*/ 88 | void set_contained(const AzDataPool *patches, 89 | AzIntArr *ia_contained) const 90 | { 91 | ia_contained->reset(); 92 | int px; 93 | for (px = 0; px < patches->size(); ++px) { 94 | const Az2D *pch = patches->point(px); 95 | double center_x = pch->x0 + (double)pch->xsz/2; 96 | double center_y = pch->y0 + (double)pch->ysz/2; 97 | if (contains(center_x, center_y)) { 98 | ia_contained->put(px); 99 | } 100 | } 101 | } 102 | 103 | inline bool contains(double xx, double yy) const { 104 | if (xx >= x0 && xx < x1 && 105 | yy >= y0 && yy < y1) { 106 | return true; 107 | } 108 | return false; 109 | } 110 | 111 | protected: 112 | void sync() { 113 | x1=x0+xsz; 114 | y1=y0+ysz; 115 | } 116 | }; 117 | #endif -------------------------------------------------------------------------------- /src/nnet/AzCuda_PmatSpa.cuh: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzCuda_PmatSpa.cuh 3 | * Copyright (C) 2015 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZ_CUDA_PMAT_SPA_CUH_ 20 | #define _AZ_CUDA_PMAT_SPA_CUH_ 21 | 22 | #include "AzUtil.hpp" 23 | #include "AzP.h" 24 | 25 | void azc2call_prod_dense1_sparse0( 26 | AzFloat *dst, int r_num, int c_num, 27 | const AzFloat *src1, int r_num1, int c_num1, /* dense */ 28 | const AzFloat *csc_vals, const int *csc_ptrs, const int *csc_rows, bool do_add); 29 | void azc2call_prod_sparse0_dense1( 30 | AzFloat *dst, int r_num, int c_num, 31 | const AzFloat *csr_vals, 32 | const int *nzrow_ptrs, const int *nzrow_rows, int nzrow_num, /* only for nonzero rows */ 33 | const int *csr_cols, 34 | const AzFloat *src2, int r_num2, int c_num2, /* dense */ 35 | bool do_add); 36 | void azc2call_prod_sparse0_dense1_a( 37 | AzFloat *dst, int r_num, int c_num, 38 | const AzFloat *csr_vals, 39 | const int *nzrow_ptrs, const int *nzrow_rows, int nzrow_num, /* only for nonzero rows */ 40 | const int *csr_cols, 41 | const AzFloat *src2, int r_num2, int c_num2, /* dense */ 42 | AzFloat alpha, bool do_add); 43 | void azc2call_add_sparse( 44 | AzFloat *dst, int r_num, int c_num, 45 | const AzFloat *csc_vals, const int *csc_rows, const int *csc_cols, int vals_num, 46 | AzFloat coeff); 47 | #endif -------------------------------------------------------------------------------- /src/nnet/AzMultiConn.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzMultiConn.hpp 3 | * Copyright (C) 2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZ_MULTI_CONN_HPP_ 20 | #define _AZ_MULTI_CONN_HPP_ 21 | 22 | #include "AzUtil.hpp" 23 | #include "AzStrPool.hpp" 24 | #include "AzParam.hpp" 25 | #include "AzPrint.hpp" 26 | 27 | /** 28 | * - Layer#0 is always the first layer. 29 | * - The last layer is always the top layer. 30 | **/ 31 | /*------------------------------------------------------------*/ 32 | class AzMultiConn { 33 | public: 34 | int lsz; 35 | AzIIarr iia_conn; 36 | AzStrPool sp_conn; 37 | AzIntArr ia_order; 38 | AzDataArr aia_below, aia_above; 39 | bool do_add, do_show_nicknm; 40 | bool is_multi; /* set by resetParam. false if there is no conn parameter(s). */ 41 | AzDataArr aia_orelease; /* to save memory: 11/26/2016 */ 42 | /* static const int version = 0; */ 43 | static const int version = 1; /* for read|write_compact of sp_conn: 6/20/2017 */ 44 | static const int reserved_len = 64; 45 | 46 | public: 47 | AzMultiConn() : do_add(true), is_multi(false), do_show_nicknm(true) {} 48 | void reset() { 49 | iia_conn.reset(); 50 | sp_conn.reset(); 51 | ia_order.reset(); 52 | aia_below.reset(); 53 | aia_above.reset(); 54 | aia_orelease.reset(); 55 | } 56 | void write(AzFile *file) const { 57 | AzTools::write_header(file, version, reserved_len); 58 | file->writeBool(is_multi); 59 | file->writeBool(do_add); 60 | file->writeInt(lsz); 61 | iia_conn.write(file); 62 | sp_conn.write_compact(file); 63 | ia_order.write(file); 64 | aia_below.write(file); 65 | aia_above.write(file); 66 | } 67 | void read(AzFile *file) { 68 | int my_version = AzTools::read_header(file, reserved_len); 69 | is_multi = file->readBool(); 70 | do_add = file->readBool(); 71 | lsz = file->readInt(); 72 | iia_conn.read(file); 73 | if (my_version == 0) sp_conn.read(file); 74 | else sp_conn.read_compact(file); 75 | ia_order.read(file); 76 | aia_below.read(file); 77 | aia_above.read(file); 78 | } 79 | 80 | virtual bool is_multi_conn() const { return is_multi; } 81 | virtual void setup(bool is_warmstart) { 82 | order_layers(lsz, iia_conn, ia_order, aia_below, aia_above); 83 | insert_connectors(ia_order, aia_below, aia_above); 84 | reset_for_output_release(); 85 | } 86 | virtual bool is_additive() const { return do_add; } 87 | 88 | virtual void resetParam(AzParam &azp, int hid_num, bool is_warmstart=false); 89 | virtual void printParam(const AzOut &out) const; 90 | 91 | virtual const AzIntArr &order() const { return ia_order; } 92 | virtual int below(int lx) { 93 | check_lay_ind(lx, "AzMultiConn::below"); 94 | if (aia_below[lx]->size() <= 0) return -1; 95 | return (*aia_below[lx])[0]; 96 | } 97 | virtual int above(int lx) { 98 | check_lay_ind(lx, "AzMultiConn::above"); 99 | if (aia_above[lx]->size() <= 0) return -1; 100 | return (*aia_above[lx])[0]; 101 | } 102 | virtual const AzIntArr &all_below(int lx) { 103 | check_ind(lx, "AzMultiConn::all_below"); 104 | return (*aia_below[lx]); 105 | } 106 | virtual const AzIntArr &all_above(int lx) { 107 | check_ind(lx, "AzMultiConn::all_above"); 108 | return (*aia_above[lx]); 109 | } 110 | 111 | static void show_below_above(const AzIntArr &ia_below, const AzIntArr &ia_above, AzBytArr &s); 112 | 113 | template void release_output(int curr_lx, AzDataArr &amat) const { 114 | if (aia_orelease.size() <= 0) return; 115 | const AzIntArr *ia = aia_orelease[curr_lx]; 116 | for (int ix = 0; ix < ia->size(); ++ix) amat((*ia)[ix])->reset(); 117 | } 118 | 119 | protected: 120 | virtual void check_lay_ind(int lx, const char *eyec) const { 121 | AzX::throw_if(lx < 0 || lx >= lsz, eyec, "out of range (layers)"); 122 | } 123 | virtual void check_ind(int lx, const char *eyec) const { 124 | AzX::throw_if(lx < 0 || lx >= ia_order.size(), eyec, "out of range (layers and connectors)"); 125 | } 126 | 127 | virtual void _parse_binary_conns(int hid_num, AzParam &azp, AzIIarr &iia_conn, AzStrPool &sp_conn) const; 128 | virtual void _parse_conns(int hid_num, AzParam &azp, AzIIarr &iia_conn, AzStrPool &sp_conn) const; 129 | virtual int parse_layerno(const char *str, int hid_num) const; 130 | virtual void order_layers(int layer_num, const AzIIarr &iia_conn, 131 | /*--- output ---*/ 132 | AzIntArr &ia_order, AzDataArr &aia_below, AzDataArr &aia_above) const; 133 | virtual void insert_connectors(AzIntArr &ia_order, AzDataArr &aia_below, AzDataArr &aia_above) const; /* inout */ 134 | void reset_for_output_release(); 135 | }; 136 | #endif 137 | -------------------------------------------------------------------------------- /src/nnet/AzP.h: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzP.h 3 | * Copyright (C) 2014-2016 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZ_P_H_ 20 | #define _AZ_P_H_ 21 | #ifdef __AZ_GPU__ 22 | #include 23 | #include 24 | #include 25 | #include 26 | #define azc_kernel(f,a,b) f<<>> 27 | #define azc_thno (blockDim.x * blockIdx.x + threadIdx.x) 28 | #define azc_thnum (blockDim.x * gridDim.x) 29 | #else 30 | #include "AzP_cpu.h" 31 | #endif 32 | 33 | #ifdef __AZ_DOUBLE__ 34 | typedef double AzFloat; 35 | #define azc_exp_arg_min (-700) 36 | #define azc_exp_arg_max (700) 37 | #define azc_epsilon 1e-307 38 | #else 39 | typedef float AzFloat; 40 | #define azc_exp_arg_min (-80) 41 | #define azc_exp_arg_max (80) 42 | #define azc_epsilon 1e-37 43 | #endif 44 | 45 | /*--- used only for GPU though ---*/ 46 | #define azc_numShared 4096 47 | 48 | /* c: column, e: ptr to elements, r: the number of rows */ 49 | #define _column(c,e,r) ((e)+(c)*(r)) 50 | /* r: row, c: column, p: ptr to elements, rn: the number of rows */ 51 | #define _entry(r,c,p,rn) (*((p)+(c)*(rn)+(r))) 52 | #define myexp(x) exp(MAX(azc_exp_arg_min,MIN(azc_exp_arg_max,(x)))) 53 | #define azc_exp_mask(x) ( ((x) < azc_exp_arg_min || (x) > azc_exp_arg_max) ? 0 : 1 ) 54 | #endif -------------------------------------------------------------------------------- /src/nnet/AzPmatSpa_gpu.cu: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzPmatSpa_gpu.cuh 3 | * Copyright (C) 2015 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #include "AzDmat.hpp" 20 | #include "AzPmat_gpu.cuh" 21 | #include "AzMemTempl.hpp" 22 | 23 | #include "AzPmatSpa_gpu.cuh" 24 | 25 | #ifdef __AZ_DOUBLE__ 26 | /*--- double-precision ---*/ 27 | #define cusparseXcsrmm cusparseDcsrmm 28 | #define cusparseXcsrmm2 cusparseDcsrmm2 29 | #define cusparseXcsr2csc cusparseDcsr2csc 30 | #else 31 | /*--- single-precision ---*/ 32 | #define cusparseXcsrmm cusparseScsrmm 33 | #define cusparseXcsrmm2 cusparseScsrmm2 34 | #define cusparseXcsr2csc cusparseScsr2csc 35 | #endif 36 | 37 | extern AzPdevice dev; 38 | #define cusparse_handle dev.cusparse_handle 39 | #define cusparse_desc dev.cusparse_desc 40 | 41 | /*------------------------------------------------*/ 42 | void _AzPmatSpa::_csc2csr(int r_num, int c_num, 43 | const AzFloat *csc_vals, const int *csc_ptrs, const int *csc_rows, int vals_num, /* input */ 44 | AzFloat *csr_vals, int *csr_ptrs, int *csr_cols) /* output */ 45 | const 46 | { 47 | cusparseStatus_t ret = 48 | cusparseXcsr2csc(cusparse_handle, c_num, r_num, /* b/c using csr2csc to convert csc into csr */ 49 | vals_num, 50 | csc_vals, csc_ptrs, csc_rows, 51 | csr_vals, csr_cols, csr_ptrs, /* NOTE: order! */ 52 | CUSPARSE_ACTION_NUMERIC, CUSPARSE_INDEX_BASE_ZERO); 53 | 54 | AzCuda::sync_check_error_if_debug("_AzPmatSpa::_csc2csr", 0, 0); 55 | AzCuda::throwIfsparseError(ret, "_AzPmatSpa::_csc2csr", "cuspasreXcsr2csc"); 56 | } 57 | 58 | /*------------------------------------------------*/ 59 | /* cusparse: C = alpha * op(A) * B + beta * C (A is csr, B is dense) */ 60 | void _AzPmatSpa::_prod_csr_dense( 61 | AzFloat *dst, int r_num, int c_num, 62 | const AzFloat *csr_vals, const int *csr_ptrs, const int *csr_cols, int vals_num, /* sparse */ 63 | const AzFloat *src1, int r_num1, int c_num1, /* dense */ 64 | AzFloat alpha, AzFloat beta) 65 | const 66 | { 67 | cusparseOperation_t op = CUSPARSE_OPERATION_NON_TRANSPOSE; 68 | int m = r_num, n = c_num, k = r_num1; 69 | cusparseStatus_t ret = cusparseXcsrmm(cusparse_handle, op, 70 | m, n, k, vals_num, &alpha, cusparse_desc, 71 | csr_vals, csr_ptrs, csr_cols, 72 | src1, r_num1, &beta, 73 | dst, r_num); 74 | AzCuda::sync_check_error_if_debug("_AzPmatSpa::_prod_csr_dense", 0, 0); 75 | AzCuda::throwIfsparseError(ret, "_AzPmatSpa::_prod_csr_dense0", "cusparseXcsrmm"); 76 | } 77 | 78 | #ifdef __AZ_CSRMM2__ 79 | /*------------------------------------------------*/ 80 | /* cusparse: C = alpha * op(A) * op(B) + beta * C (A is csr, B is dense) */ 81 | void _AzPmatSpa::_prod_csr_dense_mm2( 82 | bool do_tran0, bool do_tran1, 83 | AzFloat *dst, int r_num, int c_num, 84 | const AzFloat *csr_vals, const int *csr_ptrs, const int *csr_cols, int vals_num, /* sparse */ 85 | const AzFloat *src1, int r_num1, int c_num1, /* dense */ 86 | AzFloat alpha, AzFloat beta) 87 | const 88 | { 89 | cusparseOperation_t op0 = (do_tran0) ? CUSPARSE_OPERATION_TRANSPOSE : CUSPARSE_OPERATION_NON_TRANSPOSE; 90 | cusparseOperation_t op1 = (do_tran1) ? CUSPARSE_OPERATION_TRANSPOSE : CUSPARSE_OPERATION_NON_TRANSPOSE; 91 | int m = r_num, n = c_num, k = r_num1; 92 | cusparseStatus_t ret = cusparseXcsrmm2(cusparse_handle, op0, op1, 93 | m, n, k, vals_num, &alpha, cusparse_desc, 94 | csr_vals, csr_ptrs, csr_cols, 95 | src1, r_num1, &beta, 96 | dst, r_num); 97 | AzCuda::sync_check_error_if_debug("_AzPmatSpa::_prod_csr_dense_mm2", 0, 0); 98 | AzCuda::throwIfsparseError(ret, "_AzPmatSpa::_prod_csr_dense0", "cusparseXcsrmm2"); 99 | } 100 | #endif 101 | -------------------------------------------------------------------------------- /src/nnet/AzPmatSpa_gpu.cuh: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzPmatSpa_gpu.cuh 3 | * Copyright (C) 2015 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZ_PMAT_SPA_GPU_CUH_ 20 | #define _AZ_PMAT_SPA_GPU_CUH_ 21 | 22 | #include 23 | #include 24 | #include "AzCuda.cuh" 25 | #include "AzCuda_PmatSpa.cuh" 26 | 27 | class _AzPmatSpa { 28 | public: 29 | /*--- cusparse ---*/ 30 | void _prod_csr_dense( 31 | AzFloat *dst, int r_num, int c_num, 32 | const AzFloat *csr_vals, const int *csr_ptrs, const int *csr_cols, int vals_num, /* sparse */ 33 | const AzFloat *src1, int r_num1, int c_num1, /* dense */ 34 | AzFloat alpha, AzFloat beta) const; 35 | void _csc2csr(int r_num, int c_num, 36 | const AzFloat *csc_vals, const int *csc_ptrs, const int *csc_rows, int vals_num, /* input */ 37 | AzFloat *csr_vals, int *csr_ptrs, int *csr_cols) const; /* output */ 38 | 39 | #ifdef __AZ_CSRMM2__ 40 | void _prod_csr_dense_mm2(bool do_tran0, bool do_tran1, 41 | AzFloat *dst, int r_num, int c_num, 42 | const AzFloat *csr_vals, const int *csr_ptrs, const int *csr_cols, int vals_num, /* sparse */ 43 | const AzFloat *src1, int r_num1, int c_num1, /* dense */ 44 | AzFloat alpha, AzFloat beta) const; 45 | #endif 46 | 47 | /*--- not cusparse ---*/ 48 | void _prod_dense1_sparse0( 49 | AzFloat *dst, int r_num, int c_num, 50 | const AzFloat *src1, int r_num1, int c_num1, /* dense */ 51 | const AzFloat *csc_vals, const int *csc_ptrs, const int *csc_rows, bool do_add) const { 52 | azc2call_prod_dense1_sparse0(dst, r_num, c_num, src1, r_num1, c_num1, csc_vals, csc_ptrs, csc_rows, do_add); 53 | } 54 | void _prod_sparse0_dense1( 55 | AzFloat *dst, int r_num, int c_num, 56 | const AzFloat *csr_vals, const int *nzrow_ptrs, const int *nzrow_rows, int nzrow_num, const int *csr_cols, 57 | const AzFloat *src2, int r_num2, int c_num2, bool do_add) const { /* dense */ 58 | azc2call_prod_sparse0_dense1(dst, r_num, c_num, csr_vals, nzrow_ptrs, nzrow_rows, nzrow_num, 59 | csr_cols, src2, r_num2, c_num2, do_add); 60 | } 61 | void _prod_sparse0_dense1_a( 62 | AzFloat *dst, int r_num, int c_num, 63 | const AzFloat *csr_vals, const int *nzrow_ptrs, const int *nzrow_rows, int nzrow_num, const int *csr_cols, 64 | const AzFloat *src2, int r_num2, int c_num2, AzFloat alpha, bool do_add) const { /* dense */ 65 | azc2call_prod_sparse0_dense1_a(dst, r_num, c_num, csr_vals, nzrow_ptrs, nzrow_rows, nzrow_num, 66 | csr_cols, src2, r_num2, c_num2, alpha, do_add); 67 | } 68 | void _add_sparse(AzFloat *dst, int r_num, int c_num, 69 | const AzFloat *csc_vals, const int *csc_rows, const int *csc_cols, int vals_num, 70 | AzFloat coeff) const { 71 | azc2call_add_sparse(dst, r_num, c_num, csc_vals, csc_rows, csc_cols, vals_num, coeff); 72 | } 73 | }; 74 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpActiv_.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpActiv_.hpp 3 | * Copyright (C) 2014-2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_ACTIV__HPP_ 20 | #define _AZP_ACTIV__HPP_ 21 | 22 | #include "AzpCompo_.hpp" 23 | #include "AzPmat.hpp" 24 | 25 | /*------------------------------------------------------------*/ 26 | class AzpActiv_ : public virtual AzpCompo_ { 27 | public: 28 | virtual ~AzpActiv_() {} 29 | virtual AzpActiv_ *clone() const = 0; 30 | virtual void reset(const AzOut &out) = 0; 31 | virtual void setup_no_activation() = 0; 32 | virtual void upward(bool is_test, AzPmat *m) = 0; 33 | virtual void upward2(AzPmat *m) = 0; 34 | virtual void downward(AzPmat *m) = 0; 35 | virtual void updateDelta(int d_num) {} 36 | virtual void flushDelta() {} 37 | virtual void end_of_epoch() {} 38 | virtual void release_ld() = 0; 39 | 40 | virtual int output_channels(int node_num) const { return node_num; } 41 | virtual void show_stat(AzBytArr &s) const {} 42 | virtual bool is_param_trained() const { return false; } 43 | virtual void copy_trained_params_from(const AzpActiv_ *) { 44 | AzX::no_support(true, "AzpActiv_::copy_trained_params", "copying trained parameters"); 45 | } 46 | virtual void set_default_type(const char *type) { 47 | AzX::no_support(true, "AzpActiv_::set_default_type", "set default type"); 48 | } 49 | }; 50 | 51 | class AzpActiv_sel_ { 52 | public: 53 | virtual AzpActiv_ *activ(const AzOut &out, AzParam &azp, const char *pfx, bool is_warmstart=false) = 0; 54 | virtual AzpActiv_ *activ(const AzOut &out, AzParam &azp, const char *dflt_pfx, const char *pfx, bool is_warmstart=false) = 0; 55 | }; 56 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpCompoSetDflt.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpCompoSetDflt.hpp 3 | * Copyright (C) 2014,2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_COMPO_SET_DFLT_HPP_ 20 | #define _AZP_COMPO_SET_DFLT_HPP_ 21 | 22 | #include "AzpCompoSet_.hpp" 23 | #include "AzpLossDflt.hpp" 24 | #include "AzpPatchDflt.hpp" 25 | #include "AzpPatchDflt_var.hpp" 26 | #include "AzpWeightDflt.hpp" 27 | #include "AzpActivDflt.hpp" 28 | #include "AzpDropoutDflt.hpp" 29 | #include "AzpPoolingDflt.hpp" 30 | #include "AzpPoolingDflt_var.hpp" 31 | #include "AzpResNormDflt.hpp" 32 | #include "AzpWeight_sgdlike.hpp" 33 | #include "AzpLmAdaD.hpp" 34 | #include "AzpLmRmsp.hpp" 35 | 36 | class AzpCompoSetDflt : public virtual AzpCompoSet_ { 37 | protected: 38 | AzpLossDflt loss_dflt; 39 | AzpPatchDflt patch_dflt; 40 | AzpPatchDflt_var patch_var_dflt; 41 | AzpDropoutDflt dropout_dflt; 42 | AzpPoolingDflt pool_dflt; 43 | AzpPoolingDflt_var pool_var_dflt; 44 | AzpResNormDflt resnorm_dflt; 45 | AzpActivDflt activ_dflt; 46 | 47 | AzpWeightDflt weight_dflt; 48 | AzpWeight_sgdlike weight_adad; 49 | AzpWeight_sgdlike weight_rmsp; 50 | 51 | AzpWeight_ *p_weight; 52 | 53 | public: 54 | AzpCompoSetDflt() : p_weight(&weight_dflt) {} 55 | 56 | #define kw_do_adadelta "AdaDelta" 57 | #define kw_do_rmsp "Rmsp" 58 | #define kw_opt "optim=" 59 | #define kw_sgd "SGD" 60 | void reset(AzParam &azp, bool for_test, const AzOut &out) { 61 | /*--- optimization algorithm ---*/ 62 | p_weight = &weight_dflt; 63 | if (!for_test) { 64 | AzBytArr s_opt(kw_sgd); 65 | if (azp.isOn(kw_do_adadelta)) s_opt.reset(kw_do_adadelta); /* for compatibility */ 66 | else if (azp.isOn(kw_do_rmsp )) s_opt.reset(kw_do_rmsp); /* for compatibility */ 67 | AzPrint o(out); azp.vStr(o, kw_opt, s_opt); 68 | AzStrPool sp(kw_do_adadelta, kw_do_rmsp, kw_sgd); 69 | AzXi::check_input(s_opt.c_str(), &sp, "AzpCompoSetDflt::reset", kw_opt); 70 | if (s_opt.equals(kw_do_adadelta)) p_weight = &weight_adad; 71 | else if (s_opt.equals(kw_do_rmsp)) p_weight = &weight_rmsp; 72 | } 73 | } 74 | 75 | virtual const AzpLoss_ *loss() const { return &loss_dflt; } 76 | virtual const AzpPatch_ *patch() const { return &patch_dflt; } 77 | virtual const AzpPatch_var_ *patch_var() const { return &patch_var_dflt; } 78 | virtual const AzpDropout_ *dropout() const { return &dropout_dflt; } 79 | virtual const AzpPooling_ *pool() const { return &pool_dflt; } 80 | virtual const AzpPooling_var_ *pool_var() const { return &pool_var_dflt; } 81 | virtual const AzpResNorm_ *resnorm() const { return &resnorm_dflt; } 82 | virtual const AzpActiv_ *activ() const { return &activ_dflt; } 83 | 84 | virtual const AzpWeight_ *weight() const { return p_weight; } 85 | }; 86 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpCompo_.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpCompo_.hpp 3 | * Copyright (C) 2014,2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_COMPO__HPP_ 20 | #define _AZP_COMPO__HPP_ 21 | 22 | #include "AzParam.hpp" 23 | #include "AzHelp.hpp" 24 | 25 | /*------------------------------------------------------------*/ 26 | class AzpCompo_ { 27 | public: 28 | virtual ~AzpCompo_() {} 29 | virtual void resetParam(const AzOut &out, AzParam &azp, const AzPfx &pfx, 30 | bool is_warmstart=false) { /* use for_testonly if it's dropout */ 31 | resetParam(azp, pfx, is_warmstart); printParam(out, pfx.pfx()); 32 | } 33 | virtual void printParam(const AzOut &out, const AzPfx &pfx) const = 0; 34 | virtual void resetParam(AzParam &azp, const AzPfx &pfx, bool is_warmstart=false) = 0; 35 | virtual void printHelp(AzHelp &h) const = 0; 36 | virtual void write(AzFile *file) const = 0; 37 | virtual void read(AzFile *file) = 0; 38 | 39 | /*--- for legacy code ---*/ 40 | virtual void resetParam(const AzOut &out, AzParam &azp, const char *_pfx0, const char *_pfx1, /* for Azplayer */ 41 | bool is_warmstart=false) { 42 | resetParam(azp, _pfx0, _pfx1, is_warmstart); printParam(out, _pfx1); 43 | } 44 | virtual void resetParam(AzParam &azp, const char *_pfx0, const char *_pfx1, bool is_warmstart=false) { /* for AzpLayer */ 45 | AzPfx pfx(_pfx0, _pfx1); resetParam(azp, pfx, is_warmstart); 46 | } 47 | virtual void printParam(const AzOut &out, const char *_pfx) const { 48 | AzPfx pfx(_pfx); printParam(out, pfx); 49 | } 50 | }; 51 | #endif 52 | 53 | -------------------------------------------------------------------------------- /src/nnet/AzpDataSeq.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpDataSeq.hpp 3 | * Copyright (C) 2014,2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_DATA_SEQ_HPP_ 20 | #define _AZP_DATA_SEQ_HPP_ 21 | 22 | #include "AzUtil.hpp" 23 | #include "AzTools.hpp" 24 | #include "AzDmat.hpp" 25 | 26 | class AzpDataSeq { 27 | protected: 28 | AzIntArr ia_seq; 29 | bool dont_shuffle, do_adjust, do_force_fast_shuffle; 30 | AzDataArr aia_cl_dxs; 31 | 32 | public: 33 | AzpDataSeq() : dont_shuffle(false), do_adjust(false), do_force_fast_shuffle(false) {} 34 | 35 | /*------------------------------------------------------------*/ 36 | #define kw_dont_shuffle "PreserveDataOrder" 37 | #define kw_do_adjust "AdjustDataOrder" 38 | #define kw_do_force_fast_shuffle "FastShuffle" 39 | void resetParam(AzParam &azp) { 40 | azp.swOn(&dont_shuffle, kw_dont_shuffle); 41 | azp.swOn(&do_adjust, kw_do_adjust); 42 | if (!dont_shuffle && !do_adjust) azp.swOn(&do_force_fast_shuffle, kw_do_force_fast_shuffle); 43 | } 44 | void printParam(const AzOut &out) const { 45 | AzPrint o(out); 46 | o.printSw(kw_dont_shuffle, dont_shuffle); 47 | o.printSw(kw_do_adjust, do_adjust); 48 | o.printSw(kw_do_force_fast_shuffle, do_force_fast_shuffle); 49 | } 50 | 51 | /*------------------------------------------------------------*/ 52 | template 53 | void init(const T *data, const AzOut &out) { 54 | aia_cl_dxs.reset(); 55 | if (!do_adjust) return; 56 | 57 | data->gen_membership(aia_cl_dxs); 58 | AzBytArr s("AzpDataSeq::init: class distribution -- "); 59 | for (int cx = 0; cx < aia_cl_dxs.size(); ++cx) s << aia_cl_dxs[cx]->size() << " "; 60 | AzPrint::writeln(out, s); 61 | } 62 | 63 | /*------------------------------------------------------------*/ 64 | template 65 | const int *gen(const T *data, int data_size, bool dont_shuffle_override=false) { 66 | const char *eyec = "AzDataSeq::gen"; 67 | ia_seq.reset(); 68 | if (do_adjust && data != NULL) { 69 | if (data->batchNum() > 1) { 70 | data->gen_membership(aia_cl_dxs); /* since it's likely to be a new batch ... */ 71 | } 72 | int class_num = aia_cl_dxs.size(); 73 | AzX::throw_if((class_num <= 0), eyec, "call init first"); 74 | if (!dont_shuffle && !dont_shuffle_override) { 75 | for (int cx = 0; cx < class_num; ++cx) AzTools::shuffle2(*aia_cl_dxs(cx)); 76 | } 77 | ia_seq.prepare(data_size); 78 | for (int ix = 0; ; ++ix) { 79 | for (int cx = 0; cx < class_num; ++cx) { 80 | const AzIntArr *ia_dxs = aia_cl_dxs.point(cx); 81 | int jx = ix % ia_dxs->size(); 82 | ia_seq.put(ia_dxs->get(jx)); 83 | } 84 | if (ia_seq.size() >= data_size) break; 85 | } 86 | } 87 | else { 88 | ia_seq.range(0, data_size); 89 | if (!dont_shuffle) { 90 | /* "shuffle2" is faster but, keep "shuffle" for compatibility with the published results */ 91 | if (do_force_fast_shuffle || data_size >= 26000000) AzTools::shuffle2(ia_seq); 92 | else AzTools::shuffle(-1, &ia_seq); 93 | } 94 | } 95 | return ia_seq.point(); 96 | } 97 | }; 98 | 99 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpDataSetDflt.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpDataSetDflt.hpp 3 | * Copyright (C) 2014,2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_DATASET_DFLT_HPP_ 20 | #define _AZP_DATASET_DFLT_HPP_ 21 | #include "AzpData_.hpp" 22 | #include "AzpData_img.hpp" 23 | #include "AzpData_imgbin.hpp" 24 | #include "AzpData_sparse.hpp" 25 | #include "AzpData_sparse_multi.hpp" 26 | 27 | class AzpDataSetDflt : public virtual AzpDataSet_ { 28 | protected: 29 | AzpData_img img; 30 | AzpData_imgbin imgbin; 31 | AzpData_sparse sparse_no_no; 32 | AzpData_sparse sparse_no_bc; 33 | AzpData_sparse sparse_bc_no; 34 | AzpData_sparse sparse_bc_bc; 35 | 36 | AzpData_sparse_multi sparse_multi_no_no; 37 | AzpData_sparse_multi sparse_multi_no_bc; 38 | AzpData_sparse_multi sparse_multi_bc_no; 39 | AzpData_sparse_multi sparse_multi_bc_bc; 40 | public: 41 | virtual void printHelp(AzHelp &h, bool do_train, bool do_test, bool is_there_y) const { 42 | AzpDataSet_::printHelp(h, do_train, do_test, is_there_y); 43 | h.item_required(kw_datatype, "Dataset type. \"sparse\" | \"img\" | \"imgbin\". "); 44 | /* img.printHelp(h, do_train, is_there_y); */ 45 | sparse_no_no.printHelp(h, is_there_y); 46 | /* sparse_multi.printHelp(h, do_train, is_there_y); */ 47 | } 48 | protected: 49 | #define kw_dataext "dsno" 50 | #define kw_dataext_old "data_ext" 51 | virtual const AzpData_ *ptr(AzParam &azp, const AzBytArr &s_typ, const AzBytArr &s_x_ext, const AzBytArr &s_y_ext) const { 52 | AzBytArr s_kw(kw_dataext, "0"), s_kw_old(kw_dataext_old, "0"), s_dataext; 53 | azp.vStr(s_kw.c_str(), &s_dataext, s_kw_old.c_str()); 54 | if (s_typ.compare("image") == 0) return &img; 55 | else if (s_typ.compare("imagebin") == 0) return &imgbin; 56 | else if (s_typ.compare("sparse_multi") == 0 || s_dataext.length() > 0) { 57 | if (s_x_ext.contains("bc")) { 58 | if (s_y_ext.contains("bc")) return &sparse_multi_bc_bc; 59 | else return &sparse_multi_bc_no; 60 | } 61 | else { 62 | if (s_y_ext.contains("bc")) return &sparse_multi_no_bc; 63 | else return &sparse_multi_no_no; 64 | } 65 | } 66 | else if (s_typ.compare("sparse") == 0) { 67 | if (s_x_ext.contains("bc")) { 68 | if (s_y_ext.contains("bc")) return &sparse_bc_bc; 69 | else return &sparse_bc_no; 70 | } 71 | else { 72 | if (s_y_ext.contains("bc")) return &sparse_no_bc; 73 | else return &sparse_no_no; 74 | } 75 | } 76 | else { 77 | AzX::throw_if(true, AzInputError, "AzpDataSetDflt::ptr", "Unknown data type: ", s_typ.c_str()); return NULL; 78 | } 79 | } 80 | }; 81 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpData_tmpl_.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpData_tmpl_.hpp 3 | * Copyright (C) 2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_DATA_TMPL_HPP_ 20 | #define _AZP_DATA_TMPL_HPP_ 21 | 22 | #include "AzUtil.hpp" 23 | #include "AzDic.hpp" 24 | 25 | class AzpData_tmpl_ { 26 | public: 27 | virtual int datasetNum() const = 0; 28 | virtual int xdim(int dsno=0) const = 0; 29 | virtual const AzDicc &get_dic(int dsno=0) const = 0; 30 | virtual bool is_vg_x() const = 0; 31 | virtual bool is_sparse_x() const = 0; 32 | virtual int ydim() const = 0; 33 | virtual int dimensionality() const = 0; 34 | virtual int size(int index) const = 0; 35 | virtual void signature(AzBytArr &s) const { 36 | s.reset(); 37 | s << "dim:" << dimensionality() << ";"; 38 | s << "channel:" << xdim() << ";"; 39 | for (int ix = 0; ix < dimensionality(); ++ix) { 40 | s << "size" << ix << ":" << size(ix) << ";"; 41 | } 42 | } 43 | virtual bool isSignatureCompatible(const AzBytArr &s_nn, const AzBytArr &s_data) const { 44 | if (s_nn.compare(&s_data) == 0) return true; 45 | AzStrPool sp1(10,10), sp2(10,10); 46 | AzTools::getStrings(s_nn.c_str(), ';', &sp1); 47 | AzTools::getStrings(s_data.c_str(), ';', &sp2); 48 | if (sp1.size() != sp2.size()) { 49 | return false; 50 | } 51 | for (int ix = 0; ix < sp1.size(); ++ix) { 52 | if (strcmp(sp1.c_str(ix), sp2.c_str(ix)) != 0) { 53 | AzBytArr s1; sp1.get(ix, &s1); 54 | AzBytArr s2; sp2.get(ix, &s2); 55 | if (ix == 0 && s1.length() != s2.length()) { 56 | /*--- sparse_multi with one dataset is equivalent to sparse ---*/ 57 | if (s1.beginsWith("[0]") && s2.compare(s1.point()+3, s1.length()-3) == 0) continue; 58 | if (s2.beginsWith("[0]") && s1.compare(s2.point()+3, s2.length()-3) == 0) continue; 59 | } 60 | else if (s1.beginsWith("size") && s2.beginsWith("size") && s1.endsWith("-1")) { 61 | /*--- nn is trained for variable-size input, and data is fixed-size ---*/ 62 | continue; 63 | } 64 | else return false; 65 | } 66 | } 67 | return true; 68 | } 69 | virtual void get_info(AzxD &data_dim) const { 70 | AzIntArr ia_sz; 71 | int dim = dimensionality(); 72 | for (int ix = 0; ix < dim; ++ix) ia_sz.put(size(ix)); 73 | data_dim.reset(&ia_sz); 74 | } 75 | }; 76 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpDropoutDflt.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpDropoutDflt.hpp 3 | * Copyright (C) 2014,2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_DROPOUT_DFLT_HPP_ 20 | #define _AZP_DROPOUT_DFLT_HPP_ 21 | 22 | #include "AzpDropout_.hpp" 23 | #include "AzPmatApp.hpp" 24 | 25 | /*------------------------------------------------------------*/ 26 | class AzpDropoutDflt : public virtual /* implements */ AzpDropout_ { 27 | protected: 28 | class AzpDropoutDflt_Param { 29 | protected: 30 | static const int version = 0; 31 | static const int reserved_len = 64; 32 | public: 33 | double dropout; 34 | AzpDropoutDflt_Param() : dropout(-1) {} 35 | 36 | #define kw_dropout "dropout=" 37 | void resetParam(AzParam &azp, const char *pfx, bool for_testonly) { 38 | azp.reset_prefix(pfx); 39 | if (!for_testonly) azp.vFloat(kw_dropout, &dropout); 40 | azp.reset_prefix(); 41 | } 42 | void checkParam(const char *pfx) { 43 | AzXi::invalid_input((dropout >= 1), "AzpDropoutDflt::resetParam", kw_dropout, pfx); 44 | } 45 | virtual void printParam(const AzOut &out, const char *pfx) const { 46 | AzPrint o(out, pfx); 47 | o.printV(kw_dropout, dropout); 48 | o.printEnd(); 49 | } 50 | virtual void printHelp(AzHelp &h) const { 51 | h.item(kw_dropout, "Dropout value.", "No dropout"); 52 | } 53 | void write(AzFile *file) const { 54 | AzTools::write_header(file, version, reserved_len); 55 | file->writeDouble(dropout); 56 | } 57 | void read(AzFile *file) { 58 | AzTools::read_header(file, reserved_len); 59 | dropout = file->readDouble(); 60 | } 61 | }; 62 | protected: 63 | AzPrng rng; 64 | AzpDropoutDflt_Param p; 65 | AzPmat m_mask; 66 | static const int version = 0; 67 | static const int reserved_len = 64; 68 | public: 69 | AzpDropoutDflt() {} 70 | virtual void resetParam(AzParam &azp, const AzPfx &pfx, bool is_warmstart) { 71 | for (int px=0; pxp = p; 93 | o->m_mask.set(&m_mask); 94 | return o; 95 | } 96 | virtual bool is_active() const { return (p.dropout > 0); } 97 | virtual void upward(bool is_test, AzPmat *m) { 98 | if (p.dropout <= 0) return; 99 | if (is_test) { 100 | m->multiply(1-p.dropout); 101 | } 102 | else { 103 | m_mask.reform_noinit(m->rowNum(), m->colNum()); 104 | rng.uniform_01(&m_mask); /* [0,1] */ 105 | m_mask.mark_gt((AzFloat)p.dropout); /* ([i,j] > dropout) ? 1 : 0 */ 106 | m->elm_multi(&m_mask); 107 | } 108 | } 109 | virtual void downward(AzPmat *m) { 110 | if (p.dropout <= 0) return; 111 | m_mask.shape_chk_tmpl(m, "AzpDropoutDflt::downward", "m_mask"); 112 | m->elm_multi(&m_mask); 113 | } 114 | virtual AzPrng &ref_rng() { return rng; } 115 | }; 116 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpDropout_.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpDropout_.hpp 3 | * Copyright (C) 2014,2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_DROPOUT__HPP_ 20 | #define _AZP_DROPOUT__HPP_ 21 | 22 | #include "AzpCompo_.hpp" 23 | #include "AzPmat.hpp" 24 | 25 | /*------------------------------------------------------------*/ 26 | class AzpDropout_ : public virtual AzpCompo_ { 27 | public: 28 | virtual ~AzpDropout_() {} 29 | virtual bool is_active() const = 0; 30 | virtual void reset(const AzOut &out) = 0; 31 | virtual AzpDropout_ *clone() const = 0; 32 | virtual void upward(bool is_test, AzPmat *m) = 0; 33 | virtual void downward(AzPmat *m) = 0; 34 | virtual void show_stat(AzBytArr &s) {} 35 | virtual void read(AzFile *file) = 0; 36 | virtual void write(AzFile *file) const = 0; 37 | virtual AzPrng &ref_rng() = 0; 38 | }; 39 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpLmAdaD.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpLmAdaD.hpp 3 | * Copyright (C) 2014-2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_LM_ADAD_HPP_ 20 | #define _AZP_LM_ADAD_HPP_ 21 | 22 | #include "AzpLmSgd.hpp" 23 | 24 | /* 25 | * "ADADELTA: An Adaptive Learning Rate Method", Matthew D. Zeiler, arXuvL1212,5701 26 | */ 27 | 28 | class AzpLmAdaD_Param { 29 | public: 30 | double rho, eps; 31 | double coeff; 32 | AzpLmAdaD_Param() : rho(-1), eps(-1), coeff(1) {} 33 | 34 | /*------------------------------------------------------------*/ 35 | #define kw_rho "adad_rho=" 36 | #define kw_eps "adad_epsilon=" 37 | void resetParam(AzParam &azp, const char *pfx, bool is_warmstart=false) { 38 | azp.reset_prefix(pfx); 39 | azp.vFloat(kw_rho, &rho); 40 | azp.vFloat(kw_eps, &eps); 41 | coeff = 1; 42 | azp.reset_prefix(); 43 | } 44 | 45 | void checkParam(const char *pfx) { 46 | const char *eyec = "AzpLmParam_adad::checkParam"; 47 | AzXi::throw_if_nonpositive(rho, eyec, kw_rho, pfx); 48 | AzXi::throw_if_nonpositive(eps, eyec, kw_eps, pfx); 49 | AzX::throw_if((rho >= 1), AzInputError, eyec, kw_rho, "must be no greater than 1."); 50 | } 51 | 52 | void printParam(const AzOut &out, const char *pfx) const { 53 | if (out.isNull()) return; 54 | AzPrint o(out, pfx); 55 | o.printV(kw_rho, rho); 56 | o.printV(kw_eps, eps); 57 | o.printEnd(); 58 | } 59 | }; 60 | 61 | /*--- AdaDelta ---*/ 62 | class AzpLmAdaD : public virtual AzpLmSgd { 63 | protected: 64 | AzPmat m_w_g2avg, v_i_g2avg; 65 | /* use dlt as d2avg */ 66 | 67 | public: 68 | AzpLmAdaD() {} 69 | virtual const char *description() const { return "AdaDelta"; } 70 | 71 | virtual void resetWork() { 72 | AzpLmSgd::resetWork(); 73 | m_w_g2avg.zeroOut(); v_i_g2avg.zeroOut(); 74 | } 75 | virtual void reformWork() { 76 | AzpLmSgd::reformWork(); 77 | m_w_g2avg.reform_tmpl(&m_w); v_i_g2avg.reform_tmpl(&v_i); 78 | } 79 | 80 | void reset(const AzpLmAdaD *inp) { /* copy */ 81 | AzpLmSgd::reset(inp); 82 | m_w_g2avg.set(&inp->m_w_g2avg); v_i_g2avg.set(&inp->v_i_g2avg); 83 | } 84 | 85 | void flushDelta(const AzpLmParam &p, const AzpLmAdaD_Param &pa) { 86 | if (p.dont_update()) return; 87 | if (grad_num <= 0) return; 88 | check_ws("AzpLmAdaD::flushDelta"); 89 | 90 | bool do_reg = true; 91 | adad_update(&m_w, &m_w_grad, &m_w_g2avg, &m_w_dlt, &m_w_init, 92 | p, pa, do_reg); 93 | do_reg = p.do_reg_intercept; 94 | adad_update(&v_i, &v_i_grad, &v_i_g2avg, &v_i_dlt, &v_i_init, 95 | p, pa, do_reg); 96 | 97 | if (p.reg_L2const > 0) do_l2const(p); 98 | grad_num = 0; 99 | } 100 | 101 | protected: 102 | void adad_update(AzPmat *m_weight, 103 | AzPmat *m_grad, /* input: grad */ 104 | AzPmat *m_g2avg, 105 | AzPmat *m_d2avg, 106 | AzPmat *m_init, 107 | const AzpLmParam &p, 108 | const AzpLmAdaD_Param &pa, 109 | bool do_reg) { 110 | double rho = pa.rho, eps = pa.eps; 111 | 112 | m_grad->divide(-grad_num); /* negative gradient: -g_t */ 113 | if (p.grad_clip > 0) { /* added 1/11/2016 */ 114 | m_grad->truncate(-p.grad_clip, p.grad_clip); 115 | } 116 | 117 | if (do_reg) { 118 | double dummy_eta = 1; 119 | add_reg_grad(p, dummy_eta, m_weight, m_grad, m_init); /* regularization */ 120 | } 121 | /* 122 | * E[g^2]_t = rho E[g^2]_{t-1} + (1 - rho) g_t^2 123 | * delta_t = - (RMS[delta_{t-1}] / RMS[g]_t) g_t; RMS[g]_t = sqrt(E[g^2]_t + epsilon) 124 | * E[delta^2]_t = rho E[delta^2]_{t-1} + (1-rho) delta_t^2 125 | * x_{t+1} = x_t + delta_t 126 | */ 127 | m_g2avg->add_square(rho, m_grad, 1-rho); /* E[g^2]_t = rho E[g^2]_{t-1} + (1 - rho)g_t^2 */ 128 | m_grad->scale_by_sqrt(m_d2avg, eps); /* - RMS[delta_{t-1}] g_t */ 129 | m_grad->scale_by_sqrt(m_g2avg, eps, true); /* delta_t = -(RMS[delta_{t-1}]/RMS[g]_t) g_t */ 130 | m_d2avg->add_square(rho, m_grad, 1-rho); /* E[delta^2]_t = rho E[delta^2]_{t-1} + (1-rho) delta_t^2 */ 131 | 132 | m_weight->add(m_grad, pa.coeff); 133 | if (p.weight_clip > 0) m_weight->truncate(-p.weight_clip, p.weight_clip); 134 | } 135 | }; 136 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpLmRmsp.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpLmRmsp.hpp 3 | * Copyright (C) 2016,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_LM_RMSP_HPP_ 20 | #define _AZP_LM_RMSP_HPP_ 21 | 22 | #include "AzpLmSgd.hpp" 23 | 24 | /* 25 | * RmsProp by (Tieleman & Hinton, 2012) 26 | */ 27 | 28 | class AzpLmRmsp_Param { 29 | public: 30 | double decay, eta, coeff, eps, grad_clip_after; 31 | bool do_shrink; 32 | AzpLmRmsp_Param() : eta(-1), decay(0.95), coeff(1), eps(-1), grad_clip_after(-1), do_shrink(false) {} 33 | 34 | /*------------------------------------------------------------*/ 35 | #define kw_decay "rmsprop_decay=" 36 | #define kw_rmsprop_eps "rmsprop_eps=" 37 | #define kw_grad_clip_after "grad_clip_after=" 38 | #define kw_do_shrink "Shrink" 39 | void resetParam(AzParam &azp, const char *pfx, bool is_warmstart=false) { 40 | azp.reset_prefix(pfx); 41 | azp.vFloat(kw_decay, &decay); 42 | azp.vFloat(kw_eta, &eta); 43 | azp.vFloat(kw_rmsprop_eps, &eps); 44 | eps = MAX(eps, azc_epsilon); 45 | azp.vFloat(kw_grad_clip_after, &grad_clip_after); 46 | azp.swOn(&do_shrink, kw_do_shrink); 47 | azp.reset_prefix(); 48 | } 49 | 50 | void checkParam(const char *pfx) { 51 | const char *eyec = "AzpLmRmsp_Param::checkParam"; 52 | AzXi::throw_if_nonpositive(decay, eyec, kw_decay, pfx); 53 | AzXi::throw_if_nonpositive(eta, eyec, kw_eta, pfx); 54 | AzX::throw_if((decay >= 1), AzInputError, eyec, kw_decay, "must be no greater than 1."); 55 | } 56 | 57 | void printParam(const AzOut &out, const char *pfx) const { 58 | if (out.isNull()) return; 59 | AzPrint o(out, pfx); 60 | o.printV(kw_eta, eta); 61 | o.printV(kw_decay, decay); 62 | o.printV(kw_rmsprop_eps, eps); 63 | o.printV(kw_grad_clip_after, grad_clip_after); 64 | o.printSw(kw_do_shrink, do_shrink); 65 | o.printEnd(); 66 | } 67 | }; 68 | 69 | class AzpLmRmsp : public virtual AzpLmSgd { 70 | protected: 71 | AzPmat m_w_g2avg, v_i_g2avg; 72 | 73 | public: 74 | AzpLmRmsp() {} 75 | virtual const char *description() const { return "Rmsp"; } 76 | 77 | virtual void resetWork() { 78 | AzpLmSgd::resetWork(); 79 | m_w_g2avg.zeroOut(); v_i_g2avg.zeroOut(); 80 | } 81 | virtual void reformWork() { 82 | AzpLmSgd::reformWork(); 83 | m_w_g2avg.reform_tmpl(&m_w); v_i_g2avg.reform_tmpl(&v_i); 84 | } 85 | 86 | void reset(const AzpLmRmsp *inp) { /* copy */ 87 | AzpLmSgd::reset(inp); 88 | m_w_g2avg.set(&inp->m_w_g2avg); v_i_g2avg.set(&inp->v_i_g2avg); 89 | } 90 | 91 | void flushDelta(const AzpLmParam &p, const AzpLmRmsp_Param &pa) { 92 | if (p.dont_update()) return; 93 | if (grad_num <= 0) return; 94 | check_ws("AzpLmRmsp::flushDelta"); 95 | 96 | bool do_reg = true; 97 | rmsp_update(&m_w, &m_w_grad, &m_w_g2avg, &m_w_init, p, pa, do_reg); 98 | do_reg = p.do_reg_intercept; 99 | rmsp_update(&v_i, &v_i_grad, &v_i_g2avg, &v_i_init, p, pa, do_reg); 100 | 101 | if (p.reg_L2const > 0) do_l2const(p); 102 | grad_num = 0; 103 | } 104 | 105 | protected: 106 | void rmsp_update(AzPmat *m_weight, 107 | AzPmat *m_grad, /* input: grad */ 108 | AzPmat *m_g2avg, 109 | AzPmat *m_init, 110 | const AzpLmParam &p, 111 | const AzpLmRmsp_Param &pa, 112 | bool do_reg) { 113 | double rho = pa.decay, eta = pa.eta*pa.coeff; 114 | 115 | m_grad->divide(-grad_num); /* negative gradient: -g_t */ 116 | if (do_reg && !pa.do_shrink) add_reg_grad(p, 1, m_weight, m_grad, m_init); /* regularization */ 117 | if (p.grad_clip > 0) m_grad->truncate(-p.grad_clip, p.grad_clip); 118 | /* 119 | * r_t = (1-decay) g^2_t + decay r_{t-1} 120 | * delta_t = -eta / sqrt(r_t + epsilon) g # epsilon for stability 121 | * x_{t+1} = x_t + delta_t 122 | */ 123 | 124 | m_g2avg->add_square(rho, m_grad, 1-rho); 125 | m_grad->scale_by_sqrt(m_g2avg, pa.eps, true); 126 | if (do_reg && pa.do_shrink) add_reg_grad(p, 1, m_weight, m_grad, m_init); /* shrink weights */ 127 | if (pa.grad_clip_after > 0) m_grad->truncate(-pa.grad_clip_after, pa.grad_clip_after); 128 | m_weight->add(m_grad, eta); 129 | if (p.weight_clip > 0) m_weight->truncate(-p.weight_clip, p.weight_clip); 130 | } 131 | }; 132 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpLmSgd.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpLmSgd.hpp 3 | * Copyright (C) 2014-2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_LM_SGD_HPP_ 20 | #define _AZP_LM_SGD_HPP_ 21 | 22 | #include "AzUtil.hpp" 23 | #include "AzPmat.hpp" 24 | #include "AzpLm.hpp" 25 | 26 | class AzpLmSgd_Param { 27 | public: 28 | double eta, etab_coeff, momentum; 29 | bool do_fast_flush; 30 | AzpLmSgd_Param() : eta(-1), etab_coeff(1), momentum(-1), do_fast_flush(true) {} 31 | 32 | /*------------------------------------------------------------*/ 33 | #define kw_momentum "momentum=" 34 | #define kw_eta "step_size=" 35 | #define kw_etab_coeff "step_sizeb_coeff=" 36 | #define kw_do_fast_flush "FastFlush" 37 | #define kw_no_fast_flush "NoFastFlush" 38 | void resetParam(AzParam &azp, const char *pfx, bool is_warmstart=false) { 39 | azp.reset_prefix(pfx); 40 | azp.vFloat(kw_eta, &eta); 41 | azp.vFloat(kw_etab_coeff, &etab_coeff); 42 | azp.vFloat(kw_momentum, &momentum); 43 | if (do_fast_flush) azp.swOff(&do_fast_flush, kw_no_fast_flush); 44 | else azp.swOn(&do_fast_flush, kw_do_fast_flush); 45 | azp.reset_prefix(); 46 | } 47 | 48 | void checkParam(const char *pfx) { 49 | const char *eyec = "AzpLmSgd_Param::checkParam"; 50 | AzXi::throw_if_nonpositive(eta, eyec, kw_eta, pfx); 51 | AzXi::throw_if_nonpositive(etab_coeff, eyec, kw_etab_coeff, pfx); 52 | } 53 | 54 | void printParam(const AzOut &out, const char *pfx) const { 55 | if (out.isNull()) return; 56 | AzPrint o(out, pfx); 57 | o.printV(kw_eta, eta); 58 | o.printV(kw_etab_coeff, etab_coeff); 59 | o.printV(kw_momentum, momentum); 60 | o.printSw(kw_do_fast_flush, do_fast_flush); 61 | o.printEnd(); 62 | } 63 | void printHelp(AzHelp &h) const { 64 | h.item_required(kw_eta, "Step-size (learning rate) for SGD."); 65 | h.item(kw_momentum, "Momentum for SGD."); 66 | } 67 | }; 68 | 69 | class AzpLmSgd : public virtual AzpLm { 70 | protected: 71 | bool do_gradpart; 72 | AzPmat m_w_grad, m_w_dlt; 73 | AzPmat v_i_grad, v_i_dlt; 74 | 75 | int grad_num; 76 | 77 | public: 78 | AzpLmSgd() : grad_num(0), do_gradpart(false) {} 79 | virtual const char *description() const { return "SGD"; } 80 | virtual void resetWork() { 81 | m_w_grad.zeroOut(); m_w_dlt.zeroOut(); 82 | v_i_grad.zeroOut(); v_i_dlt.zeroOut(); 83 | grad_num = 0; 84 | } 85 | virtual void reformWork() { 86 | m_w_grad.reform_tmpl(&m_w); m_w_dlt.reform_tmpl(&m_w); 87 | v_i_grad.reform_tmpl(&v_i); v_i_dlt.reform_tmpl(&v_i); 88 | grad_num = 0; 89 | } 90 | virtual void clearTemp() { clearGrad(); } 91 | 92 | void reset(const AzpLmSgd *inp) { /* copy */ 93 | AzpLm::reset(inp); 94 | m_w_grad.set(&inp->m_w_grad); m_w_dlt.set(&inp->m_w_dlt); 95 | v_i_grad.set(&inp->v_i_grad); v_i_dlt.set(&inp->v_i_dlt); 96 | grad_num = inp->grad_num; 97 | } 98 | 99 | virtual void updateDelta(int d_num, const AzpLmParam &p, 100 | const AzPmatSpa *m_x, const AzPmat *m_deriv, const void *pp=NULL) { 101 | AzX::throw_if_null(m_x, m_deriv, "AzpLmSgd::updateDelta(spa)"); 102 | if (grad_num <= 0) _updateDelta(d_num, p, m_x, m_deriv); 103 | else _updateDelta2(d_num, p, m_x, m_deriv); 104 | } 105 | virtual void updateDelta(int d_num, const AzpLmParam &p, 106 | const AzPmat *m_x, const AzPmat *m_deriv, const void *pp=NULL) { 107 | AzX::throw_if_null(m_deriv, "AzpLmSgd::updateDelta(dense)"); 108 | if (m_x == NULL) _updateDelta(d_num, p, m_deriv); /* 12/04/2016 for bn */ 109 | else if (grad_num <= 0) _updateDelta(d_num, p, m_x, m_deriv); 110 | else _updateDelta2(d_num, p, m_x, m_deriv); 111 | } 112 | virtual void flushDelta(const AzpLmParam &p, const AzpLmSgd_Param &ps); 113 | 114 | protected: 115 | virtual void clearGrad() { m_w_grad.zeroOut(); v_i_grad.zeroOut(); grad_num=0; } 116 | template 117 | void _updateDelta(int d_num, const AzpLmParam &p, const M *m_x, const AzPmat *m_deriv); 118 | template 119 | void _updateDelta2(int d_num, const AzpLmParam &p, const M *m_x, const AzPmat *m_deriv); 120 | void _updateDelta(int d_num, const AzpLmParam &p, const AzPmat *m_grad); 121 | virtual void regularize(const AzpLmParam &p, double eta, double etab); 122 | virtual void add_reg_grad(const AzpLmParam &p, double eta, const AzPmat *m, 123 | AzPmat *m_delta, /* output */ 124 | const AzPmat *m_init=NULL) /* optional input */ const; 125 | virtual void do_l2const(const AzpLmParam &p); 126 | }; 127 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpLoss_.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpLoss_.hpp 3 | * Copyright (C) 2014,2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_LOSS__HPP_ 20 | #define _AZP_LOSS__HPP_ 21 | 22 | #include "AzParam.hpp" 23 | #include "AzHelp.hpp" 24 | #include "AzPmat.hpp" 25 | #include "AzpData_.hpp" 26 | 27 | /*------------------------------------------------------------*/ 28 | class AzpLoss_ { 29 | public: 30 | virtual ~AzpLoss_() {} 31 | virtual AzpLoss_ *clone() const = 0; 32 | virtual void resetParam(const AzOut &out, AzParam &azp, bool is_warmstart=false) = 0; 33 | virtual void printParam(const AzOut &out) const = 0; 34 | virtual void printHelp(AzHelp &h) const = 0; 35 | virtual void check_losstype(const AzOut &out, int class_num) const = 0; 36 | virtual void check_target(const AzOut &out, const AzpData_ *data) const = 0; 37 | 38 | /*--- ---*/ 39 | virtual void get_loss_deriv(const AzPmat *m_p, /* output of the top layer, col: data points */ 40 | const AzpData_ *data, 41 | const int *dxs, int d_num, 42 | /*--- output ---*/ 43 | AzPmat *m_loss_deriv, /* row: nodes of the output layer, col: data points */ 44 | double *loss, /* may be NULL: added */ 45 | const AzPmatSpa *_ms_y=NULL, 46 | const AzPmat *_md_y=NULL) const = 0; 47 | 48 | virtual double get_loss(const AzpData_ *data, int dx_begin, int d_num, const AzPmat *m_p) const = 0; 49 | 50 | virtual bool needs_eval_at_once() const = 0; 51 | #define AzpEvalNoSupport "NO-SUPPORT" 52 | virtual double test_eval(const AzpData_ *data, 53 | AzPmat *m_p, /* inout */ 54 | double *out_loss, /* output */ 55 | const AzOut &out, 56 | AzBytArr *s_pf=NULL /* description of the returned performance (e.g., "err") */ 57 | ) const = 0; 58 | virtual void test_eval2(const AzpData_ *data, int dx, int d_num, const AzPmat *m_p, /* input */ 59 | double &perf_sum, int &num, double *out_loss, AzBytArr *s_pf=NULL) /* added */ 60 | const = 0; 61 | 62 | virtual void read(AzFile *file) = 0; 63 | virtual void write(AzFile *file) const = 0; 64 | 65 | /*--- for auto encoder ---*/ 66 | virtual void get_loss_deriv(const AzPmat *m_p, const AzPmat *m_y, 67 | AzPmat *m_loss_deriv, 68 | double *loss) const = 0; /* may be NULL: added */ 69 | virtual double get_loss(const AzPmat *m_p, const AzPmat *m_y) const = 0; 70 | }; 71 | #endif 72 | -------------------------------------------------------------------------------- /src/nnet/AzpMain_reNet.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpMain_reNet.hpp 3 | * Copyright (C) 2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_MAIN_RENET_HPP_ 20 | #define _AZP_MAIN_RENET_HPP_ 21 | 22 | #include 23 | #include "AzUtil.hpp" 24 | #include "AzPrint.hpp" 25 | #include "AzpDataSetDflt.hpp" 26 | #include "AzParam.hpp" 27 | #include "AzpCompoSetDflt.hpp" 28 | 29 | #include "AzpReNet.hpp" 30 | 31 | /*------------------------------------------------------------*/ 32 | class AzpMain_reNet_Param_ { 33 | public: 34 | bool doLog, doDump; 35 | 36 | AzpMain_reNet_Param_() : doLog(true), doDump(false) {} 37 | virtual void resetParam(const AzOut &out, AzParam &azp) = 0; 38 | virtual void printHelp(AzHelp &h) const { 39 | h.writeln("No parameter!"); 40 | } 41 | virtual void reset(AzParam &azp, const AzOut &out, const AzBytArr &s_action) { 42 | if (azp.needs_help()) { 43 | printHelp(out); 44 | AzX::throw_if(true, AzNormal, "", ""); 45 | } 46 | print_hline(out); 47 | AzPrint::writeln(out, s_action, " ", azp.c_str()); 48 | print_hline(out); 49 | resetParam(out, azp); 50 | } 51 | /*------------------------------------------------------------*/ 52 | void print_hline(const AzOut &out) const { 53 | if (out.isNull()) return; 54 | AzPrint::writeln(out, "--------------------"); 55 | } 56 | protected: 57 | void printHelp(const AzOut &out) const { AzHelp h(out); printHelp(h); h.end(); } 58 | void _resetParam(AzPrint &o, AzParam &azp); 59 | static void _printHelp(AzHelp &h); 60 | void setupLogDmp(AzPrint &o, AzParam &azp); 61 | }; 62 | 63 | /*------------------------------------------------------------*/ 64 | class AzpMain_reNet { 65 | protected: 66 | AzpCompoSetDflt cs; 67 | virtual AzpReNet *alloc_renet_for_test(AzObjPtrArr &opa, AzParam &azp) { 68 | return alloc_renet(opa, azp, true); 69 | } 70 | virtual AzpReNet *alloc_renet(AzObjPtrArr &opa, AzParam &azp, bool for_test=false) { 71 | opa.alloc(1); 72 | cs.reset(azp, for_test, log_out); 73 | opa.set(0, new AzpReNet(&cs)); 74 | return opa(0); 75 | } 76 | 77 | public: 78 | AzpMain_reNet() {} 79 | 80 | void renet(int argc, const char *argv[], const AzBytArr &s_action); 81 | void predict(int argc, const char *argv[], const AzBytArr &s_action); 82 | void write_word_mapping(int argc, const char *argv[], const AzBytArr &s_action); 83 | void write_embedded(int argc, const char *argv[], const AzBytArr &s_action); 84 | 85 | protected: 86 | void _write_embedded(AzpReNet *net, AzParam &azp, const AzpData_ *tst, int mb, int feat_top_num, const char *fn); 87 | AzOut *reset_eval(const AzBytArr &s_eval_fn, AzOfs &ofs, AzOut &eval_out); 88 | }; 89 | #endif 90 | -------------------------------------------------------------------------------- /src/nnet/AzpPatch_.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpPatch_.hpp 3 | * Copyright (C) 2014-2015 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_PATCH__HPP_ 20 | #define _AZP_PATCH__HPP_ 21 | 22 | #include "AzxD.hpp" 23 | #include "AzPmat.hpp" 24 | #include "AzpCompo_.hpp" 25 | 26 | /* generate patches/filters for convolutional NNs */ 27 | class AzpPatch_ : public virtual AzpCompo_ { 28 | public: 29 | virtual ~AzpPatch_() {} 30 | virtual AzpPatch_ *clone() const = 0; 31 | virtual bool is_convolution() const = 0; 32 | virtual void reset(const AzOut &out, const AzxD *input, int channels, bool is_spa, bool is_var) = 0; 33 | virtual void setup_allinone(const AzOut &out, const AzxD *input, int channels) = 0; 34 | virtual void setup_asis(const AzOut &out, const AzxD *input, int channels) = 0; 35 | virtual int get_channels() const = 0; 36 | virtual bool isSameInput(const AzxD *inp, int channels) { 37 | return (inp != NULL && inp->isSame(input_region()) && get_channels() && channels); 38 | } 39 | virtual const AzxD *input_region(AzxD *o=NULL) const = 0; 40 | virtual const AzxD *output_region(AzxD *o=NULL) const = 0; 41 | virtual int patch_length() const = 0; /* size times channels */ 42 | 43 | virtual void show_input(const char *header, const AzOut &out) const {} 44 | virtual void show_output(const char *header, const AzOut &out) const {} 45 | 46 | virtual void upward(bool is_test, 47 | const AzPmat *m_bef, /* each column represents a pixel; more than one data point */ 48 | AzPmat *m_aft) const = 0; /* each column represents a patch */ 49 | virtual void downward(const AzPmat *m_aft, AzPmat *m_bef) const = 0; 50 | }; 51 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpPatch_var_.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpPatch_var_.hpp 3 | * Copyright (C) 2014-2015 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_PATCH_VAR__HPP_ 20 | #define _AZP_PATCH_VAR__HPP_ 21 | 22 | #include "AzxD.hpp" 23 | #include "AzPmat.hpp" 24 | #include "AzpCompo_.hpp" 25 | 26 | /* generate patches/filters for convolutional NNs */ 27 | class AzpPatch_var_ : public virtual AzpCompo_ { 28 | public: 29 | virtual ~AzpPatch_var_() {} 30 | virtual AzpPatch_var_ *clone() const = 0; 31 | virtual void reset(const AzOut &out, int channels, bool is_spa) = 0; 32 | virtual int patch_length() const = 0; /* size times channels */ 33 | 34 | virtual void upward(bool is_test, 35 | const AzPmatVar *m_bef, /* each column represents a pixel; more than one data point */ 36 | AzPmatVar *m_aft) = 0; /* each column represents a patch */ 37 | virtual void downward(const AzPmatVar *m_aft, AzPmatVar *m_bef) = 0; 38 | }; 39 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpPooling_.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpPooling_.hpp 3 | * Copyright (C) 2014,2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_POOLING__HPP_ 20 | #define _AZP_POOLING__HPP_ 21 | 22 | #include "AzpCompo_.hpp" 23 | #include "AzParam.hpp" 24 | #include "AzPmat.hpp" 25 | #include "AzxD.hpp" 26 | 27 | /*------------------------------------------------------------*/ 28 | class AzpPooling_ : public virtual AzpCompo_ { 29 | public: 30 | virtual ~AzpPooling_() {} 31 | virtual AzpPooling_ *clone() const = 0; 32 | virtual void reset(const AzOut &out, const AzxD *input, AzxD *output) = 0; 33 | virtual int input_size() const = 0; 34 | virtual int output_size() const = 0; 35 | virtual void output(AzxD *output) = 0; 36 | virtual void upward(bool is_test, const AzPmat *m_inp, AzPmat *m_out) = 0; 37 | virtual void downward(const AzPmat *m_lossd_after, AzPmat *m_lossd_before) const = 0; 38 | virtual void show(const AzOut &out) const {}; 39 | virtual void get_chosen(AzIntArr *ia_chosen) const = 0; /* for analysis */ 40 | virtual bool do_asis() const = 0; 41 | 42 | /*--- just for convenience ... ---*/ 43 | virtual void upward(bool is_test, AzPmat *m_inout) { 44 | if (do_asis()) return; 45 | AzPmat m_out; upward(is_test, m_inout, &m_out); 46 | m_inout->set(&m_out); 47 | } 48 | virtual void downward(AzPmat *m_inout) const { 49 | if (do_asis()) return; 50 | AzPmat m_inp(m_inout); downward(&m_inp, m_inout); 51 | } 52 | }; 53 | #endif 54 | 55 | -------------------------------------------------------------------------------- /src/nnet/AzpPooling_var_.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpPooling_var_.hpp 3 | * Copyright (C) 2014,2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_POOLING_VAR__HPP_ 20 | #define _AZP_POOLING_VAR__HPP_ 21 | 22 | #include "AzpCompo_.hpp" 23 | #include "AzPmat.hpp" 24 | #include "AzPmatApp.hpp" 25 | 26 | /**************************************************************/ 27 | /* from variable size to fixed size */ 28 | class AzpPooling_var_ : public virtual AzpCompo_ { 29 | public: 30 | virtual ~AzpPooling_var_() {} 31 | virtual AzpPooling_var_ *clone() const = 0; 32 | virtual void reset(const AzOut &out) = 0; 33 | virtual void output(AzxD *out) const = 0; 34 | virtual void upward(bool is_test, const AzPmatVar *m_inp, AzPmatVar *m_out) = 0; 35 | virtual void downward(const AzPmatVar *m_lossd_after, AzPmatVar *m_lossd_before) const = 0; 36 | virtual const AzIntArr *out_ind() const = 0; 37 | virtual void get_chosen(AzIntArr *ia_chosen) const = 0; /* for analysis */ 38 | virtual bool do_asis() const = 0; 39 | 40 | /*--- just for convenience ... ---*/ 41 | virtual void upward(bool is_test, AzPmatVar *mv_inout) { 42 | if (do_asis()) return; 43 | AzPmatVar mv_out; upward(is_test, mv_inout, &mv_out); 44 | mv_inout->set(&mv_out); 45 | } 46 | virtual void downward(AzPmatVar *m_inout) const { 47 | if (do_asis()) return; 48 | AzPmatVar m_inp(m_inout); downward(&m_inp, m_inout); 49 | } 50 | }; 51 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpResNorm_.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpResNorm_.hpp 3 | * Copyright (C) 2014,2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_RESNORM__HPP_ 20 | #define _AZP_RESNORM__HPP_ 21 | 22 | #include "AzUtil.hpp" 23 | #include "AzPmat.hpp" 24 | #include "AzParam.hpp" 25 | #include "AzpCompo_.hpp" 26 | 27 | /**********************************************************************************/ 28 | class AzpResNorm_ : public virtual AzpCompo_ { 29 | public: 30 | virtual ~AzpResNorm_() {} 31 | virtual void reset(const AzOut &out, const AzxD *input, int cc) = 0; 32 | virtual AzpResNorm_ *clone() const = 0; 33 | virtual void upward(bool is_test, AzPmat *m) = 0; 34 | virtual void downward(AzPmat *m) = 0; 35 | }; 36 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpWeight_.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpWeight_.hpp 3 | * Copyright (C) 2014,2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_WEIGHT__HPP_ 20 | #define _AZP_WEIGHT__HPP_ 21 | 22 | #include "AzpCompo_.hpp" 23 | #include "AzPmat.hpp" 24 | #include "AzpLm.hpp" 25 | 26 | /**********************************************************/ 27 | class AzpWeight_ : public virtual AzpCompo_ { 28 | public: 29 | virtual AzpWeight_ *clone() const = 0; 30 | virtual ~AzpWeight_() {} 31 | virtual void destroy() = 0; 32 | 33 | virtual bool are_weights_fixed() const = 0; 34 | 35 | /*--- do something with parameters ... ---*/ 36 | virtual void force_no_intercept() = 0; 37 | virtual void reset_do_no_intercept(bool flag) = 0; 38 | virtual void force_no_reg() = 0; /* force no regularization */ 39 | virtual void multiply_to_stepsize(double factor, const AzOut *out=NULL) = 0; 40 | virtual void set_momentum(double newmom, const AzOut *out=NULL) = 0; 41 | 42 | virtual void force_thru(int inp_dim) = 0; 43 | 44 | /*--- initialization ---*/ 45 | virtual void reset(int loc_num, int w_num, int inp_dim, bool is_spa, bool is_var) = 0; 46 | virtual void setup_for_reg_L2init() = 0; 47 | virtual void check_for_reg_L2init() const = 0; 48 | virtual void initWeights() = 0; 49 | virtual void initWeights(const AzpLm *inp, double coeff) = 0; 50 | 51 | /*--- up and down ... ---*/ 52 | virtual void upward(bool is_test, const AzPmat *m_x, AzPmat *m_out) = 0; 53 | virtual void upward(bool is_test, const AzPmatSpa *m_x, AzPmat *m_out) = 0; 54 | virtual void downward(const AzPmat *m_lossd, AzPmat *m_d) const = 0; 55 | virtual void updateDelta(int d_num, const AzPmat *m_x, const AzPmat *m_lossd) = 0; 56 | virtual void updateDelta(int d_num, const AzPmatSpa *m_x, const AzPmat *m_lossd) = 0; 57 | virtual void flushDelta() = 0; /* prev is used in SVRG */ 58 | virtual void clearTemp() = 0; 59 | virtual void end_of_epoch() = 0; 60 | 61 | /*--- seeking information ... ---*/ 62 | virtual double regloss(double *iniloss) const = 0; 63 | virtual int get_dim() const = 0; 64 | virtual int classNum() const = 0; 65 | virtual const AzpLm *linmod() const = 0; 66 | virtual AzpLm *linmod_u() = 0; 67 | 68 | virtual void reset_monitor() {} 69 | virtual int num_weights() const = 0; 70 | virtual void show_stat(AzBytArr &s) const {} 71 | }; 72 | 73 | #endif -------------------------------------------------------------------------------- /src/nnet/AzpWeight_sgdlike.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzpWeight_sgdlike.hpp 3 | * Copyright (C) 2015-2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZP_WEIGHT_SGDLIKE_HPP_ 20 | #define _AZP_WEIGHT_SGDLIKE_HPP_ 21 | 22 | #include "AzpWeightDflt.hpp" 23 | 24 | /**********************************************************/ 25 | template 26 | class AzpWeight_sgdlike : public virtual AzpWeightDflt { 27 | protected: 28 | AzDataArr lmods_opt; 29 | OptParam pa; 30 | 31 | virtual void reset_lms() { 32 | lms_const.realloc(lmods_opt.size(), "AzpWeight_sgdlike::reset_lms", "lms_const"); 33 | lms.realloc(lmods_opt.size(), "AzpWeight_sgdlike::reset_lms", "lms"); 34 | for (int lx = 0; lx < lmods_opt.size(); ++lx) { 35 | lms_const(lx, lmods_opt[lx]); 36 | lms(lx, lmods_opt(lx)); 37 | } 38 | } 39 | 40 | public: 41 | AzpWeight_sgdlike() {} 42 | virtual void reset(int loc_num, /* not used */ 43 | int w_num, int inp_dim, bool is_spa, bool is_var) { 44 | int sz = _reset_common(loc_num, w_num, inp_dim, is_spa, is_var); 45 | if (do_thru) return; 46 | lmods_sgd.reset(); 47 | lmods_opt.reset(sz); 48 | for (int lx = 0; lx < lmods_opt.size(); ++lx) ((AzpLm *)lmods_opt(lx))->reset(w_num, p, inp_dim); 49 | reset_lms(); 50 | } 51 | virtual AzpWeight_ *clone() const { 52 | AzpWeight_sgdlike *o = new AzpWeight_sgdlike(); 53 | o->lmods_opt.reset(&lmods_opt); 54 | o->reset_lms(); 55 | o->p = p; 56 | o->pa = pa; 57 | o->do_thru = do_thru; 58 | o->thru_dim = thru_dim; 59 | return o; 60 | } 61 | virtual void read(AzFile *file) { 62 | read_common(file); 63 | lmods_opt.read(file); /* AzpLm::read */ reset_lms(); 64 | } 65 | virtual void write(AzFile *file) const { 66 | write_common(file); 67 | lmods_opt.write(file); /* AzpLm::write */ 68 | } 69 | 70 | /*------------------------------------------------------------*/ 71 | virtual void _resetParam(AzParam &azp, const char *pfx, bool is_warmstart=false) { 72 | _resetParam_common(azp, pfx, is_warmstart); 73 | pa.resetParam(azp, pfx, is_warmstart); 74 | } 75 | virtual void printParam(const AzOut &out, const AzPfx &pfx) const { 76 | _printParam_common(out, pfx.pfx()); 77 | pa.printParam(out, pfx.pfx()); 78 | } 79 | virtual void resetParam(AzParam &azp, const AzPfx &pfx, bool is_warmstart=false) { 80 | for (int px=0; pxflushDelta(p, pa); 95 | } 96 | }; 97 | 98 | #endif -------------------------------------------------------------------------------- /src/nnet/AzxD.hpp: -------------------------------------------------------------------------------- 1 | /* * * * * 2 | * AzxD.hpp 3 | * Copyright (C) 2014,2015,2017 Rie Johnson 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | * * * * */ 18 | 19 | #ifndef _AZ_XD_HPP_ 20 | #define _AZ_XD_HPP_ 21 | 22 | #include "AzUtil.hpp" 23 | #include "AzPrint.hpp" 24 | 25 | class AzxD { 26 | protected: 27 | AzIntArr ia_sz; /* size of dim0, dim1, dim2, ... */ 28 | 29 | public: 30 | AzxD() {} 31 | AzxD(const AzxD *inp) { ia_sz.reset(&inp->ia_sz); } 32 | AzxD(int xsz, int ysz) { reset(xsz, ysz); } 33 | virtual bool is_var() const { return !is_valid(); } 34 | virtual bool is_valid() const { 35 | if (ia_sz.size() <= 0) return false; 36 | int ix; 37 | for (ix = 0; ix < ia_sz.size(); ++ix) if (ia_sz.get(ix) <= 0) return false; 38 | return true; 39 | } 40 | virtual void reset(int dim=0) { /* generate a unit size region */ 41 | ia_sz.reset(); 42 | if (dim == 0) return; 43 | ia_sz.reset(dim, 1); 44 | } 45 | virtual bool isSame(const AzxD *inp) const { 46 | if (ia_sz.compare(&inp->ia_sz) != 0) return false; 47 | return true; 48 | } 49 | virtual void reset(const AzxD *inp) { 50 | ia_sz.reset(&inp->ia_sz); 51 | } 52 | virtual void reset(const AzIntArr *ia) { 53 | ia_sz.reset(ia); 54 | } 55 | virtual void reset(const int *arr, int len) { 56 | ia_sz.reset(arr, len); 57 | } 58 | virtual void reset(int xsz, int ysz) { 59 | ia_sz.reset(); ia_sz.put(xsz); ia_sz.put(ysz); 60 | } 61 | 62 | virtual int get_dim() const { return ia_sz.size(); } 63 | 64 | virtual int sz(int dx) const { 65 | AzX::throw_if((dx < 0 || dx >= ia_sz.size()), "AzxD::sz", "dim is out of range"); 66 | return ia_sz.get(dx); 67 | } 68 | 69 | virtual int get_min_size() const { 70 | int dim = get_dim(); 71 | int minsz = -1; 72 | int dx; 73 | for (dx = 0; dx < dim; ++dx) { 74 | int mysz = sz(dx); 75 | if (dx == 0 || mysz < minsz) minsz = mysz; 76 | } 77 | return minsz; 78 | } 79 | virtual int size() const { 80 | int dim = get_dim(); 81 | if (dim == 0) return 0; 82 | 83 | int region_size = 1; 84 | int dx; 85 | for (dx = 0; dx < dim; ++dx) { 86 | int mysz = sz(dx); 87 | region_size *= mysz; 88 | } 89 | return region_size; 90 | } 91 | void write(AzFile *file) const { 92 | ia_sz.write(file); 93 | } 94 | void read(AzFile *file) { 95 | ia_sz.read(file); 96 | } 97 | inline virtual void show(const char *header, const AzOut &out) const { 98 | AzBytArr s; 99 | s.c(header); 100 | format(s); 101 | AzPrint::writeln(out, s); 102 | } 103 | virtual void format(AzBytArr &s, bool do_reset=false) const { 104 | if (do_reset) s.reset(); 105 | // s.c("("); 106 | int ix; 107 | for (ix = 0; ix < ia_sz.size(); ++ix) { 108 | if (ix > 0) s.c(" x "); 109 | s.cn(ia_sz.get(ix)); 110 | } 111 | // s.c(")"); 112 | } 113 | }; 114 | #endif -------------------------------------------------------------------------------- /src/nnet/driv_reNet.cpp: -------------------------------------------------------------------------------- 1 | #define _AZ_MAIN_ 2 | #include "AzUtil.hpp" 3 | #include "AzpMain_reNet.hpp" 4 | 5 | extern AzPdevice dev; 6 | 7 | /*-----------------------------------------------------------------*/ 8 | void help() 9 | { 10 | #ifdef __AZ_GPU__ 11 | cout << "Arguments: gpu#[:mem] action parameters" <getMessage() << endl; 66 | ret = -1; 67 | goto labend; 68 | } 69 | 70 | labend: 71 | dev.closeDevice(); 72 | return ret; 73 | } 74 | --------------------------------------------------------------------------------