├── .gitignore ├── .gitmodules ├── LICENSE ├── README.md ├── build.sbt ├── datas └── MR │ ├── list.txt │ ├── rt-polarity.neg │ └── rt-polarity.pos ├── project ├── build.properties └── site.sbt ├── run_scripts ├── train_ac_blstm.sh └── train_g_ac_blstm.sh └── src └── main └── scala └── experiments ├── AC_BLSTM_TextClassification.scala ├── DataHelper.scala ├── G_AC_BLSTM.scala └── Ops.scala /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | 19 | # Compiled Static libraries 20 | *.lai 21 | *.la 22 | *.a 23 | *.lib 24 | 25 | # Executables 26 | *.exe 27 | *.out 28 | *.app 29 | *~ 30 | 31 | # doc 32 | doc/html 33 | doc/latex 34 | doc/doc 35 | docs/web-data 36 | 37 | #dmlc 38 | config.mk 39 | 40 | *.pyc 41 | .Rhistory 42 | *log 43 | Debug 44 | *suo 45 | tracker 46 | 47 | # vim 48 | *.swp 49 | *.swo 50 | *.swn 51 | .vimrc 52 | .ycm_extra_conf.py 53 | .ycm_extra_conf.pyc 54 | .clang-format 55 | 56 | # Emacs 57 | .#* 58 | .clang_complete 59 | .dir-locals.el 60 | __pycache__ 61 | *.pkl 62 | *.params 63 | *.states 64 | *.json 65 | *.d 66 | cmake-build* 67 | data 68 | model 69 | recommonmark 70 | 71 | # R 72 | *.Rcheck 73 | *.rds 74 | *.Rproj 75 | .Rproj.user 76 | R-package/inst/* 77 | *.tar.gz 78 | *.tgz 79 | R-package/man/*.Rd 80 | R-package/R/mxnet_generated.R 81 | 82 | # data 83 | *.rec 84 | *.lst 85 | *.zip 86 | *ubyte 87 | *.bin 88 | *.txt 89 | !CMakeLists.txt 90 | 91 | # ipython notebook 92 | *_pb2.py 93 | *.ipynb_checkpoints* 94 | input.txt* 95 | 96 | # Jetbrain 97 | .idea 98 | .gradle 99 | *.iml 100 | 101 | # ctags 102 | tags 103 | 104 | # cscope 105 | cscope.out 106 | cscope.files 107 | 108 | # Eclipse project config 109 | .project 110 | .cproject 111 | .classpath 112 | .settings 113 | .pydevproject 114 | CMakeFiles 115 | cmake_install.cmake 116 | 117 | # Visual Studio Code 118 | .vscode 119 | 120 | # Mac OS X 121 | .DS_Store 122 | 123 | #Notebook Automated Test 124 | !tests/nightly/test_tutorial_config.txt 125 | !tests/nightly/TestNotebook 126 | tests/nightly/tmp_notebook 127 | 128 | # pip building tools 129 | tools/pip_package/build 130 | tools/pip_package/dist 131 | tools/pip_package/mxnet.egg-info 132 | tools/pip_package/mxnet 133 | 134 | # temporary path for building dependencies when building wheel 135 | deps/ 136 | staticdeps/ 137 | tmp/ 138 | build/ 139 | lib/ 140 | bin/ 141 | model/ 142 | 143 | # VTune 144 | ./r0*hs 145 | 146 | # generated function signature for IDE auto-complete 147 | python/mxnet/symbol/gen_* 148 | python/mxnet/ndarray/gen_* 149 | python/.eggs 150 | 151 | # tests if built insource 152 | *CTestTestfile.cmake 153 | *DartConfiguration.tcl 154 | tests/Makefile 155 | tests/mxnet_unit_tests 156 | 157 | # Code coverage related 158 | .coverage 159 | *.gcov 160 | *.gcno 161 | coverage.xml 162 | 163 | # Local CMake build config 164 | cmake_options.yml 165 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "incubator-mxnet"] 2 | path = incubator-mxnet 3 | url = https://github.com/apache/incubator-mxnet.git 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 梁德澎 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![996.ICU](https://img.shields.io/badge/link-996.icu-red.svg)](https://996.icu/#/en_US) 2 | 3 | # AC-BLSTM 4 | MXNet Scala module implementation of my work [AC-BLSTM[1]](https://arxiv.org/abs/1611.01884). 5 | 6 | # Setup 7 | ## Environment 8 | Tested on Ubuntu 14.04, using CUDA 8.0.61. 9 | 10 | ## Build Steps 11 | ### Build MXNet 12 | 13 | make -j4 USE_MKLDNN=0 USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 14 | 15 | For more details how to build MXNet from source pls refer to: http://mxnet.io/get_started/ubuntu_setup.html. 16 | 17 | #### Requirements to Build MXNet-Scala-Package 18 | * Java 8 19 | * [maven](https://maven.apache.org/download.cgi) 20 | 21 | make scalapkg 22 | 23 | For more details how to build MXNet-Scala-Package pls refer to: http://mxnet.io/get_started/ubuntu_setup.html#install-the-mxnet-package-for-scala. 24 | 25 | ### Build AC-BLSTM Project 26 | #### Requirements 27 | * [sbt 0.13](http://www.scala-sbt.org/) 28 | 29 | under the AC-BLSTM folder: 30 | ```bash 31 | mkdir lib 32 | cp mxnet/scala-package/assembly/linux-x86_64-gpu/target/mxnet-full_2.11-linux-x86_64-gpu-0.1.2-SNAPSHOT.jar lib 33 | ``` 34 | Then run `sbt` and compile the project 35 | 36 | ## Run Experiments 37 | ### Download Word2Vec Model 38 | You can download the pretrained Word2Vec Model in this url: https://code.google.com/archive/p/word2vec/, then put the 39 | `GoogleNews-vectors-negative300.bin` file to the `datas` path. 40 | 41 | ### Run Experiments 42 | #### AC-BLSTM on MR Dataset 43 | ```bash 44 | cd run_scripts 45 | bash train_ac_blstm.sh 46 | ``` 47 | #### G-AC-BLSTM on MR Dataset 48 | ```bash 49 | cd run_scripts 50 | bash train_g_ac_blstm.sh 51 | ``` 52 | 53 | Because I was doing the 10-fold cross-validation on MR dataset, so you can modify the `CROSS_VALIDATION_ID=` flag from 0 to 9 for the cross-validation expriements. 54 | 55 | By the way, If you can successfully reproduce the result reported in the paper, congratulations :) . 56 | 57 | If not, God knows what happen :( . 58 | 59 | May the force be with you :) ..... 60 | 61 | ## References 62 | [1] Liang, Depeng, and Yongdong Zhang. "AC-BLSTM: Asymmetric Convolutional Bidirectional LSTM Networks for Text Classification." arXiv preprint arXiv:1611.01884 (2016). 63 | 64 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | lazy val commonSettings = Seq( 2 | version := "1.0.0", 3 | scalaVersion := "2.11.8" 4 | ) 5 | 6 | lazy val root = (project in file(".")). 7 | settings(commonSettings: _*). 8 | settings( 9 | name := "AC-BLSTM" 10 | ) 11 | 12 | libraryDependencies ++= Seq( 13 | "args4j" % "args4j" % "2.33", 14 | "org.slf4j" % "slf4j-api" % "1.7.25" 15 | ) 16 | -------------------------------------------------------------------------------- /datas/MR/list.txt: -------------------------------------------------------------------------------- 1 | 8316 2 | 10188 3 | 2711 4 | 6353 5 | 7482 6 | 961 7 | 9338 8 | 8274 9 | 8276 10 | 3490 11 | 1231 12 | 4762 13 | 1573 14 | 4754 15 | 10080 16 | 2934 17 | 7393 18 | 1116 19 | 436 20 | 7346 21 | 326 22 | 4919 23 | 5897 24 | 824 25 | 4523 26 | 930 27 | 3851 28 | 8872 29 | 499 30 | 9960 31 | 1549 32 | 8679 33 | 2823 34 | 10332 35 | 3795 36 | 6137 37 | 3190 38 | 2311 39 | 4313 40 | 4861 41 | 6130 42 | 2462 43 | 5556 44 | 8565 45 | 2887 46 | 7649 47 | 7147 48 | 1249 49 | 9135 50 | 8354 51 | 546 52 | 4376 53 | 9638 54 | 1706 55 | 8545 56 | 7133 57 | 9681 58 | 967 59 | 4287 60 | 1570 61 | 8828 62 | 1435 63 | 8688 64 | 10198 65 | 2367 66 | 474 67 | 10648 68 | 10336 69 | 10453 70 | 6580 71 | 7132 72 | 9732 73 | 5467 74 | 4337 75 | 229 76 | 3900 77 | 6642 78 | 72 79 | 7695 80 | 7976 81 | 290 82 | 6359 83 | 1822 84 | 4152 85 | 6845 86 | 4207 87 | 1402 88 | 7430 89 | 3423 90 | 9935 91 | 5756 92 | 2401 93 | 882 94 | 149 95 | 9704 96 | 9830 97 | 5216 98 | 4322 99 | 6329 100 | 2851 101 | 7958 102 | 2192 103 | 10400 104 | 385 105 | 6102 106 | 2982 107 | 6235 108 | 2093 109 | 6558 110 | 6402 111 | 8871 112 | 9243 113 | 749 114 | 2091 115 | 4626 116 | 5449 117 | 4131 118 | 1474 119 | 592 120 | 4976 121 | 779 122 | 6533 123 | 758 124 | 603 125 | 2998 126 | 1804 127 | 4658 128 | 5096 129 | 4707 130 | 6072 131 | 859 132 | 3603 133 | 8973 134 | 9669 135 | 7691 136 | 5684 137 | 7174 138 | 2748 139 | 9331 140 | 1696 141 | 1515 142 | 1771 143 | 2522 144 | 10345 145 | 6808 146 | 481 147 | 3137 148 | 9466 149 | 3299 150 | 7146 151 | 2092 152 | 4750 153 | 2140 154 | 8200 155 | 1373 156 | 559 157 | 6977 158 | 8414 159 | 1846 160 | 1983 161 | 10169 162 | 3383 163 | 6540 164 | 5762 165 | 7037 166 | 7077 167 | 771 168 | 8131 169 | 8832 170 | 103 171 | 3831 172 | 3063 173 | 9859 174 | 9076 175 | 4855 176 | 6432 177 | 5232 178 | 1241 179 | 7986 180 | 8249 181 | 7778 182 | 6428 183 | 4253 184 | 6344 185 | 9388 186 | 4495 187 | 10572 188 | 6754 189 | 9606 190 | 8919 191 | 5240 192 | 7356 193 | 2669 194 | 1458 195 | 127 196 | 10434 197 | 1677 198 | 4538 199 | 7473 200 | 8144 201 | 3096 202 | 6310 203 | 5531 204 | 5554 205 | 7469 206 | 187 207 | 8510 208 | 421 209 | 9644 210 | 5062 211 | 8561 212 | 1744 213 | 3419 214 | 3152 215 | 3714 216 | 9572 217 | 1305 218 | 1971 219 | 10231 220 | 1335 221 | 3058 222 | 687 223 | 7831 224 | 2306 225 | 1027 226 | 3635 227 | 7178 228 | 3818 229 | 6131 230 | 8481 231 | 3048 232 | 548 233 | 8992 234 | 8517 235 | 7759 236 | 10383 237 | 1160 238 | 5694 239 | 10125 240 | 1006 241 | 1853 242 | 9108 243 | 5696 244 | 6412 245 | 5121 246 | 8275 247 | 6835 248 | 9628 249 | 5194 250 | 2361 251 | 1493 252 | 1376 253 | 761 254 | 4333 255 | 2500 256 | 5 257 | 8866 258 | 5648 259 | 5675 260 | 5364 261 | 5405 262 | 10470 263 | 10153 264 | 8878 265 | 10237 266 | 9539 267 | 5505 268 | 6444 269 | 4914 270 | 836 271 | 6766 272 | 1367 273 | 8216 274 | 3657 275 | 2815 276 | 6542 277 | 3790 278 | 8171 279 | 3398 280 | 7061 281 | 185 282 | 3267 283 | 3582 284 | 7199 285 | 9197 286 | 5135 287 | 4375 288 | 4924 289 | 7325 290 | 801 291 | 9025 292 | 4508 293 | 9228 294 | 283 295 | 4466 296 | 4999 297 | 6398 298 | 3843 299 | 1077 300 | 9119 301 | 9467 302 | 3685 303 | 10488 304 | 10507 305 | 3011 306 | 6190 307 | 2840 308 | 10263 309 | 5726 310 | 7787 311 | 5202 312 | 1851 313 | 9462 314 | 2718 315 | 9319 316 | 8404 317 | 1252 318 | 4454 319 | 4171 320 | 3322 321 | 3528 322 | 6207 323 | 10356 324 | 4913 325 | 1879 326 | 9565 327 | 1558 328 | 7544 329 | 1253 330 | 7051 331 | 7534 332 | 8505 333 | 6681 334 | 155 335 | 10254 336 | 4883 337 | 3596 338 | 5722 339 | 9875 340 | 2609 341 | 8786 342 | 495 343 | 9449 344 | 929 345 | 4862 346 | 4448 347 | 4076 348 | 2233 349 | 2826 350 | 5092 351 | 3942 352 | 2951 353 | 1333 354 | 4534 355 | 4772 356 | 6196 357 | 1562 358 | 10621 359 | 1518 360 | 7593 361 | 2105 362 | 2534 363 | 5973 364 | 3614 365 | 9101 366 | 6784 367 | 171 368 | 9963 369 | 1189 370 | 9602 371 | 9993 372 | 7720 373 | 655 374 | 6722 375 | 380 376 | 5996 377 | 5834 378 | 5623 379 | 5222 380 | 1198 381 | 7106 382 | 4705 383 | 3768 384 | 4473 385 | 4842 386 | 9798 387 | 4525 388 | 4323 389 | 5871 390 | 9384 391 | 4598 392 | 9720 393 | 9558 394 | 2434 395 | 4006 396 | 4860 397 | 8266 398 | 4666 399 | 272 400 | 2747 401 | 10277 402 | 230 403 | 2185 404 | 428 405 | 10496 406 | 3607 407 | 3494 408 | 3258 409 | 8133 410 | 614 411 | 3624 412 | 7985 413 | 8074 414 | 9555 415 | 10631 416 | 629 417 | 10167 418 | 1649 419 | 7274 420 | 705 421 | 5830 422 | 8289 423 | 7458 424 | 4415 425 | 8839 426 | 6048 427 | 5746 428 | 4052 429 | 9130 430 | 3592 431 | 3009 432 | 4700 433 | 4499 434 | 4785 435 | 1342 436 | 6413 437 | 6463 438 | 6626 439 | 7198 440 | 9224 441 | 4498 442 | 8199 443 | 7065 444 | 7951 445 | 9575 446 | 375 447 | 151 448 | 7191 449 | 7083 450 | 5720 451 | 6193 452 | 6891 453 | 1946 454 | 5349 455 | 7354 456 | 2880 457 | 640 458 | 2256 459 | 8092 460 | 3748 461 | 10620 462 | 8559 463 | 4277 464 | 1225 465 | 3116 466 | 589 467 | 2004 468 | 908 469 | 9322 470 | 7765 471 | 2159 472 | 7486 473 | 9948 474 | 6223 475 | 1977 476 | 6645 477 | 2184 478 | 713 479 | 6414 480 | 7850 481 | 2360 482 | 9936 483 | 6857 484 | 10014 485 | 5841 486 | 3395 487 | 6684 488 | 1919 489 | 9198 490 | 9547 491 | 4030 492 | 88 493 | 236 494 | 5898 495 | 1850 496 | 6439 497 | 7419 498 | 117 499 | 5903 500 | 10086 501 | 5718 502 | 6950 503 | 671 504 | 5650 505 | 4778 506 | 6383 507 | 5586 508 | 684 509 | 1938 510 | 8560 511 | 10402 512 | 6650 513 | 2349 514 | 1917 515 | 6901 516 | 9734 517 | 8840 518 | 4340 519 | 2999 520 | 6517 521 | 10578 522 | 3015 523 | 4606 524 | 1847 525 | 7707 526 | 3873 527 | 8150 528 | 2308 529 | 6654 530 | 10404 531 | 9951 532 | 607 533 | 1318 534 | 4968 535 | 4365 536 | 3021 537 | 1832 538 | 2812 539 | 8771 540 | 5499 541 | 6014 542 | 7782 543 | 8000 544 | 2151 545 | 6239 546 | 4013 547 | 3674 548 | 10397 549 | 10217 550 | 2439 551 | 48 552 | 631 553 | 9866 554 | 3516 555 | 1564 556 | 3004 557 | 6987 558 | 5311 559 | 8318 560 | 8378 561 | 4945 562 | 2922 563 | 1120 564 | 4317 565 | 1684 566 | 2482 567 | 4548 568 | 3014 569 | 4227 570 | 1008 571 | 686 572 | 5230 573 | 6372 574 | 327 575 | 7179 576 | 2502 577 | 4230 578 | 7847 579 | 8273 580 | 3304 581 | 2220 582 | 1140 583 | 8338 584 | 2278 585 | 1450 586 | 5680 587 | 3849 588 | 3390 589 | 2346 590 | 6717 591 | 1781 592 | 2024 593 | 1694 594 | 5276 595 | 9493 596 | 8094 597 | 2545 598 | 3956 599 | 5914 600 | 7566 601 | 9131 602 | 4133 603 | 1020 604 | 8603 605 | 3154 606 | 2245 607 | 4319 608 | 8242 609 | 5578 610 | 5155 611 | 10037 612 | 4422 613 | 9189 614 | 5749 615 | 7288 616 | 6459 617 | 3930 618 | 6539 619 | 278 620 | 3206 621 | 2651 622 | 6430 623 | 6495 624 | 4595 625 | 8907 626 | 9599 627 | 8190 628 | 3141 629 | 532 630 | 4649 631 | 1824 632 | 3526 633 | 4417 634 | 9845 635 | 5005 636 | 2293 637 | 3961 638 | 2057 639 | 1738 640 | 6395 641 | 8405 642 | 4145 643 | 9341 644 | 3701 645 | 6364 646 | 1552 647 | 1429 648 | 4670 649 | 45 650 | 9150 651 | 6135 652 | 3450 653 | 7229 654 | 3400 655 | 830 656 | 1113 657 | 2987 658 | 3860 659 | 4110 660 | 10196 661 | 9427 662 | 3097 663 | 8045 664 | 2358 665 | 10291 666 | 2420 667 | 2351 668 | 1988 669 | 7424 670 | 2678 671 | 3527 672 | 2193 673 | 2086 674 | 4402 675 | 9612 676 | 5317 677 | 7057 678 | 5584 679 | 1979 680 | 7599 681 | 2196 682 | 10320 683 | 324 684 | 7488 685 | 5440 686 | 7898 687 | 9191 688 | 10319 689 | 1927 690 | 8313 691 | 3652 692 | 9714 693 | 6563 694 | 7833 695 | 5797 696 | 6895 697 | 9283 698 | 3934 699 | 4728 700 | 8700 701 | 6989 702 | 7464 703 | 2820 704 | 10173 705 | 6280 706 | 3184 707 | 2965 708 | 4047 709 | 4656 710 | 206 711 | 2020 712 | 3193 713 | 9026 714 | 10168 715 | 10443 716 | 4084 717 | 4223 718 | 5159 719 | 9393 720 | 3551 721 | 6492 722 | 3242 723 | 8063 724 | 6793 725 | 6038 726 | 8862 727 | 221 728 | 3804 729 | 3241 730 | 9200 731 | 6838 732 | 10190 733 | 1857 734 | 10591 735 | 5979 736 | 2724 737 | 5486 738 | 721 739 | 10446 740 | 6997 741 | 341 742 | 7956 743 | 770 744 | 1177 745 | 2788 746 | 9389 747 | 5451 748 | 5059 749 | 9440 750 | 4661 751 | 8877 752 | 8537 753 | 1111 754 | 9739 755 | 1808 756 | 6113 757 | 8605 758 | 9334 759 | 932 760 | 6297 761 | 1272 762 | 5607 763 | 5949 764 | 1528 765 | 5700 766 | 6956 767 | 8857 768 | 9543 769 | 10109 770 | 7984 771 | 7371 772 | 3971 773 | 7227 774 | 1642 775 | 1350 776 | 8122 777 | 3744 778 | 6363 779 | 9782 780 | 672 781 | 1650 782 | 3881 783 | 2090 784 | 1873 785 | 5439 786 | 5098 787 | 10529 788 | 8698 789 | 10371 790 | 9192 791 | 3535 792 | 3676 793 | 4012 794 | 9968 795 | 1398 796 | 5882 797 | 6226 798 | 3572 799 | 5976 800 | 6221 801 | 5751 802 | 10479 803 | 4003 804 | 10084 805 | 6706 806 | 729 807 | 6267 808 | 742 809 | 520 810 | 7564 811 | 4048 812 | 6986 813 | 6066 814 | 9787 815 | 3496 816 | 6813 817 | 2255 818 | 3069 819 | 6968 820 | 10660 821 | 1377 822 | 6457 823 | 3309 824 | 10542 825 | 4312 826 | 6276 827 | 85 828 | 1105 829 | 6167 830 | 4744 831 | 6698 832 | 8754 833 | 4932 834 | 7117 835 | 6159 836 | 7654 837 | 5733 838 | 5591 839 | 10334 840 | 3658 841 | 4461 842 | 3272 843 | 1413 844 | 1673 845 | 5854 846 | 7011 847 | 4148 848 | 4237 849 | 5197 850 | 802 851 | 650 852 | 8979 853 | 1432 854 | 4978 855 | 3837 856 | 5151 857 | 5705 858 | 5557 859 | 228 860 | 5238 861 | 6617 862 | 4587 863 | 874 864 | 669 865 | 1405 866 | 4331 867 | 7332 868 | 1481 869 | 8154 870 | 8042 871 | 1795 872 | 7410 873 | 4660 874 | 7190 875 | 7498 876 | 6646 877 | 4194 878 | 133 879 | 9127 880 | 8982 881 | 7705 882 | 2443 883 | 5575 884 | 5719 885 | 6765 886 | 6988 887 | 7005 888 | 6745 889 | 10087 890 | 8522 891 | 10310 892 | 3223 893 | 4764 894 | 2180 895 | 9351 896 | 6270 897 | 10652 898 | 10467 899 | 2617 900 | 950 901 | 7478 902 | 7257 903 | 9939 904 | 1050 905 | 9498 906 | 8542 907 | 18 908 | 2511 909 | 1375 910 | 9957 911 | 3251 912 | 9735 913 | 4212 914 | 2070 915 | 145 916 | 7602 917 | 7054 918 | 7162 919 | 9245 920 | 8953 921 | 1385 922 | 8332 923 | 4926 924 | 5778 925 | 6785 926 | 3814 927 | 5498 928 | 2446 929 | 6018 930 | 7923 931 | 10072 932 | 5972 933 | 4399 934 | 4425 935 | 5571 936 | 1043 937 | 5910 938 | 1641 939 | 226 940 | 6211 941 | 8283 942 | 7799 943 | 2680 944 | 4903 945 | 6397 946 | 9585 947 | 10262 948 | 8109 949 | 7397 950 | 9413 951 | 8716 952 | 6802 953 | 6737 954 | 6083 955 | 10630 956 | 2605 957 | 1685 958 | 5351 959 | 6384 960 | 7957 961 | 10359 962 | 7791 963 | 5281 964 | 1414 965 | 3946 966 | 2866 967 | 1788 968 | 2779 969 | 6174 970 | 8238 971 | 7034 972 | 9347 973 | 6569 974 | 8745 975 | 9728 976 | 2363 977 | 4441 978 | 4974 979 | 6635 980 | 2647 981 | 9523 982 | 5309 983 | 10598 984 | 2209 985 | 3310 986 | 8027 987 | 2557 988 | 9396 989 | 3522 990 | 8225 991 | 9515 992 | 5833 993 | 6390 994 | 7977 995 | 6285 996 | 6093 997 | 3344 998 | 299 999 | 9166 1000 | 443 1001 | 4597 1002 | 5393 1003 | 2950 1004 | 5425 1005 | 5275 1006 | 5339 1007 | 5134 1008 | 2148 1009 | 9032 1010 | 10608 1011 | 43 1012 | 2060 1013 | 10624 1014 | 3585 1015 | 5767 1016 | 9415 1017 | 7204 1018 | 8447 1019 | 821 1020 | 10426 1021 | 2751 1022 | 10128 1023 | 8993 1024 | 1308 1025 | 2496 1026 | 10050 1027 | 7401 1028 | 60 1029 | 177 1030 | 1431 1031 | 881 1032 | 6550 1033 | 10259 1034 | 5946 1035 | 9487 1036 | 2703 1037 | 9579 1038 | 7238 1039 | 621 1040 | 4847 1041 | 5862 1042 | 6962 1043 | 8120 1044 | 9276 1045 | 6719 1046 | 10054 1047 | 4755 1048 | 4737 1049 | 7579 1050 | 2264 1051 | 7477 1052 | 2970 1053 | 563 1054 | 3271 1055 | 6035 1056 | 4955 1057 | 2010 1058 | 6840 1059 | 5755 1060 | 6079 1061 | 9174 1062 | 4582 1063 | 6147 1064 | 4838 1065 | 4675 1066 | 8651 1067 | 430 1068 | 4290 1069 | 1779 1070 | 4677 1071 | 2501 1072 | 6350 1073 | 768 1074 | 3805 1075 | 3458 1076 | 5395 1077 | 5034 1078 | 9453 1079 | 812 1080 | 4734 1081 | 7358 1082 | 7009 1083 | 10595 1084 | 8955 1085 | 8134 1086 | 1326 1087 | 3332 1088 | 7286 1089 | 1443 1090 | 6740 1091 | 1525 1092 | 475 1093 | 2649 1094 | 8284 1095 | 1106 1096 | 6478 1097 | 1086 1098 | 261 1099 | 8691 1100 | 7851 1101 | 7197 1102 | 7437 1103 | 9546 1104 | 8179 1105 | 203 1106 | 8642 1107 | 7021 1108 | 763 1109 | 5158 1110 | 9392 1111 | 3053 1112 | 4520 1113 | 1 1114 | 9468 1115 | 1599 1116 | 1088 1117 | 9892 1118 | 2130 1119 | 2172 1120 | 8764 1121 | 9238 1122 | 3832 1123 | 8395 1124 | 4099 1125 | 787 1126 | 6328 1127 | 9990 1128 | 5428 1129 | 7032 1130 | 7501 1131 | 287 1132 | 9346 1133 | 2307 1134 | 82 1135 | 6002 1136 | 5210 1137 | 8594 1138 | 9382 1139 | 36 1140 | 6969 1141 | 2030 1142 | 5918 1143 | 10589 1144 | 1078 1145 | 3560 1146 | 3480 1147 | 350 1148 | 5902 1149 | 9762 1150 | 2526 1151 | 7580 1152 | 1571 1153 | 9998 1154 | 5840 1155 | 2463 1156 | 10151 1157 | 6867 1158 | 4285 1159 | 7451 1160 | 5539 1161 | 10005 1162 | 2276 1163 | 3296 1164 | 1182 1165 | 5545 1166 | 8459 1167 | 4193 1168 | 4069 1169 | 3384 1170 | 2736 1171 | 3234 1172 | 7210 1173 | 10497 1174 | 3201 1175 | 2913 1176 | 6097 1177 | 9703 1178 | 7591 1179 | 7112 1180 | 316 1181 | 2602 1182 | 303 1183 | 1490 1184 | 10539 1185 | 6918 1186 | 2520 1187 | 9247 1188 | 7697 1189 | 3493 1190 | 4572 1191 | 6385 1192 | 6295 1193 | 3941 1194 | 8504 1195 | 3702 1196 | 5597 1197 | 5861 1198 | 10480 1199 | 4332 1200 | 2249 1201 | 1507 1202 | 2228 1203 | 6114 1204 | 5343 1205 | 766 1206 | 4921 1207 | 3085 1208 | 10412 1209 | 5858 1210 | 5404 1211 | 9648 1212 | 923 1213 | 9381 1214 | 2008 1215 | 9797 1216 | 10295 1217 | 2445 1218 | 8650 1219 | 3202 1220 | 5125 1221 | 10321 1222 | 9686 1223 | 8976 1224 | 46 1225 | 5066 1226 | 6584 1227 | 5117 1228 | 10225 1229 | 2544 1230 | 599 1231 | 9219 1232 | 7300 1233 | 985 1234 | 6602 1235 | 6471 1236 | 3227 1237 | 4278 1238 | 4746 1239 | 8541 1240 | 5398 1241 | 6273 1242 | 7518 1243 | 6622 1244 | 8410 1245 | 918 1246 | 5601 1247 | 3782 1248 | 9821 1249 | 10531 1250 | 4134 1251 | 7681 1252 | 3170 1253 | 4684 1254 | 485 1255 | 3612 1256 | 10637 1257 | 5906 1258 | 10490 1259 | 8709 1260 | 8513 1261 | 8544 1262 | 484 1263 | 10165 1264 | 1806 1265 | 2943 1266 | 8922 1267 | 8880 1268 | 4373 1269 | 2100 1270 | 1056 1271 | 3291 1272 | 2818 1273 | 7000 1274 | 4087 1275 | 2236 1276 | 1158 1277 | 9350 1278 | 4080 1279 | 2590 1280 | 4247 1281 | 604 1282 | 6324 1283 | 8490 1284 | 2237 1285 | 7059 1286 | 4314 1287 | 3518 1288 | 9372 1289 | 10264 1290 | 1379 1291 | 7220 1292 | 6523 1293 | 1126 1294 | 5131 1295 | 9207 1296 | 10628 1297 | 8471 1298 | 410 1299 | 3562 1300 | 4032 1301 | 9423 1302 | 6225 1303 | 9757 1304 | 7930 1305 | 5297 1306 | 9296 1307 | 10401 1308 | 1291 1309 | 168 1310 | 5845 1311 | 5254 1312 | 9375 1313 | 2983 1314 | 3589 1315 | 1053 1316 | 5054 1317 | 5004 1318 | 8475 1319 | 636 1320 | 6873 1321 | 4411 1322 | 3268 1323 | 6289 1324 | 2284 1325 | 7035 1326 | 4873 1327 | 1484 1328 | 7673 1329 | 98 1330 | 5779 1331 | 4603 1332 | 9912 1333 | 7656 1334 | 8230 1335 | 4104 1336 | 4971 1337 | 10335 1338 | 6630 1339 | 7763 1340 | 2499 1341 | 2733 1342 | 9348 1343 | 7569 1344 | 4421 1345 | 7940 1346 | 1745 1347 | 4759 1348 | 968 1349 | 5544 1350 | 6063 1351 | 5812 1352 | 1588 1353 | 137 1354 | 2215 1355 | 6993 1356 | 6189 1357 | 4629 1358 | 263 1359 | 9378 1360 | 4478 1361 | 746 1362 | 6194 1363 | 61 1364 | 251 1365 | 4692 1366 | 7508 1367 | 4601 1368 | 3317 1369 | 7459 1370 | 10326 1371 | 1337 1372 | 8152 1373 | 3555 1374 | 4463 1375 | 9280 1376 | 3777 1377 | 7665 1378 | 5032 1379 | 2337 1380 | 6815 1381 | 4809 1382 | 255 1383 | 3662 1384 | 8 1385 | 775 1386 | 2978 1387 | 387 1388 | 6939 1389 | 7443 1390 | 3139 1391 | 1107 1392 | 198 1393 | 7275 1394 | 9499 1395 | 1812 1396 | 5094 1397 | 2991 1398 | 2067 1399 | 8640 1400 | 4418 1401 | 6228 1402 | 4845 1403 | 6685 1404 | 1472 1405 | 7990 1406 | 3739 1407 | 1816 1408 | 6441 1409 | 111 1410 | 1759 1411 | 5963 1412 | 10569 1413 | 6952 1414 | 7980 1415 | 1993 1416 | 1553 1417 | 4550 1418 | 7895 1419 | 3389 1420 | 1761 1421 | 8194 1422 | 8370 1423 | 2018 1424 | 6727 1425 | 5920 1426 | 9613 1427 | 3123 1428 | 1193 1429 | 8366 1430 | 4242 1431 | 4366 1432 | 292 1433 | 7910 1434 | 9825 1435 | 1208 1436 | 3638 1437 | 7590 1438 | 9017 1439 | 9629 1440 | 4023 1441 | 9383 1442 | 9337 1443 | 4849 1444 | 5625 1445 | 6309 1446 | 8752 1447 | 1902 1448 | 6985 1449 | 1421 1450 | 5068 1451 | 8233 1452 | 4562 1453 | 7813 1454 | 595 1455 | 9630 1456 | 9253 1457 | 3921 1458 | 7139 1459 | 7435 1460 | 4524 1461 | 7298 1462 | 183 1463 | 5501 1464 | 2682 1465 | 2621 1466 | 3260 1467 | 4157 1468 | 5879 1469 | 8690 1470 | 10020 1471 | 6665 1472 | 4141 1473 | 5446 1474 | 9831 1475 | 2274 1476 | 7359 1477 | 8294 1478 | 5884 1479 | 6518 1480 | 3476 1481 | 2514 1482 | 6732 1483 | 8804 1484 | 6929 1485 | 8728 1486 | 1049 1487 | 6484 1488 | 6896 1489 | 3140 1490 | 105 1491 | 3870 1492 | 2925 1493 | 9504 1494 | 8636 1495 | 9488 1496 | 2365 1497 | 10025 1498 | 3694 1499 | 4374 1500 | 10191 1501 | 9482 1502 | 7280 1503 | 6647 1504 | 6941 1505 | 4300 1506 | 815 1507 | 2281 1508 | 7247 1509 | 6342 1510 | 5406 1511 | 10351 1512 | 1942 1513 | 5463 1514 | 782 1515 | 5367 1516 | 4057 1517 | 4703 1518 | 8634 1519 | 9568 1520 | 3633 1521 | 4251 1522 | 6 1523 | 7322 1524 | 1974 1525 | 4188 1526 | 8591 1527 | 1463 1528 | 4264 1529 | 5435 1530 | 1884 1531 | 4541 1532 | 2619 1533 | 7926 1534 | 1084 1535 | 8799 1536 | 9970 1537 | 8852 1538 | 8582 1539 | 996 1540 | 5192 1541 | 5058 1542 | 9789 1543 | 1131 1544 | 4105 1545 | 7201 1546 | 6848 1547 | 9851 1548 | 8087 1549 | 1766 1550 | 4713 1551 | 10568 1552 | 426 1553 | 9152 1554 | 9090 1555 | 5166 1556 | 4181 1557 | 503 1558 | 5935 1559 | 1295 1560 | 8457 1561 | 2532 1562 | 4741 1563 | 9457 1564 | 1787 1565 | 10535 1566 | 7911 1567 | 8903 1568 | 1139 1569 | 10521 1570 | 3835 1571 | 1185 1572 | 5253 1573 | 2885 1574 | 10285 1575 | 9340 1576 | 8626 1577 | 10017 1578 | 2119 1579 | 3357 1580 | 1827 1581 | 2000 1582 | 9886 1583 | 6612 1584 | 9265 1585 | 9677 1586 | 6880 1587 | 1125 1588 | 898 1589 | 3648 1590 | 2525 1591 | 9078 1592 | 7331 1593 | 8792 1594 | 7636 1595 | 10307 1596 | 4260 1597 | 5966 1598 | 656 1599 | 5765 1600 | 1204 1601 | 1754 1602 | 1230 1603 | 8417 1604 | 8101 1605 | 8790 1606 | 4753 1607 | 915 1608 | 453 1609 | 8530 1610 | 7743 1611 | 2116 1612 | 4514 1613 | 5122 1614 | 1262 1615 | 8089 1616 | 974 1617 | 3316 1618 | 3328 1619 | 8623 1620 | 3497 1621 | 2121 1622 | 3858 1623 | 885 1624 | 7194 1625 | 10214 1626 | 4685 1627 | 8014 1628 | 4949 1629 | 536 1630 | 9930 1631 | 9826 1632 | 7161 1633 | 4437 1634 | 271 1635 | 9456 1636 | 4202 1637 | 5658 1638 | 5929 1639 | 6306 1640 | 5388 1641 | 10502 1642 | 4698 1643 | 9634 1644 | 849 1645 | 7680 1646 | 8066 1647 | 10301 1648 | 9688 1649 | 2832 1650 | 4556 1651 | 735 1652 | 7592 1653 | 9031 1654 | 6242 1655 | 1067 1656 | 478 1657 | 4782 1658 | 8124 1659 | 8371 1660 | 10556 1661 | 4536 1662 | 776 1663 | 90 1664 | 339 1665 | 2229 1666 | 8920 1667 | 8168 1668 | 3886 1669 | 2488 1670 | 5365 1671 | 5596 1672 | 5018 1673 | 5986 1674 | 2235 1675 | 8997 1676 | 7131 1677 | 1133 1678 | 10223 1679 | 5781 1680 | 7826 1681 | 6966 1682 | 3133 1683 | 4645 1684 | 4301 1685 | 5243 1686 | 5478 1687 | 1152 1688 | 10202 1689 | 9489 1690 | 8148 1691 | 5320 1692 | 286 1693 | 9794 1694 | 5044 1695 | 712 1696 | 778 1697 | 4938 1698 | 9138 1699 | 5282 1700 | 1625 1701 | 9240 1702 | 1841 1703 | 14 1704 | 3424 1705 | 8265 1706 | 9079 1707 | 4275 1708 | 9902 1709 | 9818 1710 | 6109 1711 | 5583 1712 | 7620 1713 | 9665 1714 | 7457 1715 | 6697 1716 | 4560 1717 | 403 1718 | 8986 1719 | 2944 1720 | 6807 1721 | 4286 1722 | 7913 1723 | 6191 1724 | 5526 1725 | 6089 1726 | 1059 1727 | 7679 1728 | 7360 1729 | 9693 1730 | 6005 1731 | 3671 1732 | 9691 1733 | 905 1734 | 7655 1735 | 8543 1736 | 9920 1737 | 10251 1738 | 5954 1739 | 5587 1740 | 549 1741 | 6271 1742 | 3948 1743 | 7329 1744 | 5509 1745 | 10436 1746 | 1424 1747 | 5318 1748 | 320 1749 | 4001 1750 | 8203 1751 | 945 1752 | 1000 1753 | 4174 1754 | 3718 1755 | 7302 1756 | 4189 1757 | 1969 1758 | 2980 1759 | 2154 1760 | 4768 1761 | 3784 1762 | 7849 1763 | 1826 1764 | 425 1765 | 5201 1766 | 7099 1767 | 4696 1768 | 8298 1769 | 1224 1770 | 4492 1771 | 2692 1772 | 1364 1773 | 4309 1774 | 7026 1775 | 5140 1776 | 134 1777 | 6013 1778 | 1543 1779 | 6783 1780 | 9560 1781 | 7667 1782 | 9235 1783 | 10082 1784 | 7142 1785 | 4125 1786 | 479 1787 | 281 1788 | 3023 1789 | 8945 1790 | 1003 1791 | 7568 1792 | 5010 1793 | 9865 1794 | 4800 1795 | 1944 1796 | 3731 1797 | 8002 1798 | 2759 1799 | 315 1800 | 7935 1801 | 2962 1802 | 70 1803 | 702 1804 | 10483 1805 | 1622 1806 | 4263 1807 | 4655 1808 | 6354 1809 | 4056 1810 | 1760 1811 | 8583 1812 | 8413 1813 | 10642 1814 | 5278 1815 | 6874 1816 | 5138 1817 | 9053 1818 | 2939 1819 | 937 1820 | 8035 1821 | 6416 1822 | 6060 1823 | 8424 1824 | 5357 1825 | 9965 1826 | 1076 1827 | 2817 1828 | 4596 1829 | 6639 1830 | 9569 1831 | 9940 1832 | 3436 1833 | 7918 1834 | 5736 1835 | 5375 1836 | 7857 1837 | 10002 1838 | 6530 1839 | 8969 1840 | 5171 1841 | 947 1842 | 1620 1843 | 5568 1844 | 6082 1845 | 2328 1846 | 4552 1847 | 4727 1848 | 491 1849 | 7989 1850 | 9967 1851 | 7213 1852 | 8578 1853 | 5644 1854 | 9736 1855 | 8747 1856 | 3077 1857 | 1478 1858 | 9536 1859 | 7214 1860 | 11 1861 | 3399 1862 | 8460 1863 | 8935 1864 | 10137 1865 | 7248 1866 | 23 1867 | 2552 1868 | 1356 1869 | 7754 1870 | 3626 1871 | 1164 1872 | 4387 1873 | 3663 1874 | 6099 1875 | 10235 1876 | 899 1877 | 8034 1878 | 3187 1879 | 7237 1880 | 9813 1881 | 2917 1882 | 10415 1883 | 6905 1884 | 3 1885 | 4664 1886 | 1497 1887 | 7383 1888 | 4964 1889 | 5698 1890 | 2503 1891 | 7785 1892 | 1322 1893 | 8641 1894 | 10331 1895 | 5354 1896 | 9441 1897 | 7563 1898 | 8520 1899 | 6598 1900 | 4776 1901 | 4176 1902 | 3919 1903 | 4579 1904 | 8345 1905 | 6710 1906 | 5301 1907 | 2872 1908 | 9584 1909 | 7129 1910 | 2610 1911 | 9631 1912 | 10586 1913 | 3487 1914 | 2549 1915 | 3726 1916 | 312 1917 | 4389 1918 | 8247 1919 | 2391 1920 | 6877 1921 | 9051 1922 | 1950 1923 | 1916 1924 | 4788 1925 | 8762 1926 | 9894 1927 | 5139 1928 | 6325 1929 | 1940 1930 | 6391 1931 | 12 1932 | 3233 1933 | 2068 1934 | 2387 1935 | 1383 1936 | 7125 1937 | 2639 1938 | 10043 1939 | 4813 1940 | 2157 1941 | 1612 1942 | 7483 1943 | 4451 1944 | 5653 1945 | 3839 1946 | 1505 1947 | 8360 1948 | 5775 1949 | 8071 1950 | 7045 1951 | 8710 1952 | 2040 1953 | 3169 1954 | 5993 1955 | 10183 1956 | 582 1957 | 6996 1958 | 8575 1959 | 2464 1960 | 17 1961 | 4733 1962 | 9074 1963 | 1514 1964 | 880 1965 | 6628 1966 | 562 1967 | 5302 1968 | 527 1969 | 7608 1970 | 4432 1971 | 2221 1972 | 2979 1973 | 2798 1974 | 6236 1975 | 2345 1976 | 6576 1977 | 5304 1978 | 9982 1979 | 10554 1980 | 2675 1981 | 83 1982 | 943 1983 | 3808 1984 | 8659 1985 | 5105 1986 | 3715 1987 | 9184 1988 | 2285 1989 | 418 1990 | 9069 1991 | 1999 1992 | 5378 1993 | 983 1994 | 4213 1995 | 8898 1996 | 7554 1997 | 4405 1998 | 6951 1999 | 3936 2000 | 7812 2001 | 710 2002 | 993 2003 | 2620 2004 | 6794 2005 | 1750 2006 | 32 2007 | 3758 2008 | 5205 2009 | 1866 2010 | 3876 2011 | 8214 2012 | 260 2013 | 2611 2014 | 5333 2015 | 5212 2016 | 3792 2017 | 818 2018 | 1090 2019 | 2021 2020 | 9156 2021 | 6641 2022 | 973 2023 | 4113 2024 | 8800 2025 | 2704 2026 | 8290 2027 | 3446 2028 | 8470 2029 | 113 2030 | 10079 2031 | 30 2032 | 2223 2033 | 5790 2034 | 4140 2035 | 10247 2036 | 10308 2037 | 6846 2038 | 2642 2039 | 6064 2040 | 9256 2041 | 5261 2042 | 9600 2043 | 6183 2044 | 4083 2045 | 8703 2046 | 3584 2047 | 1646 2048 | 8734 2049 | 9039 2050 | 10276 2051 | 1469 2052 | 5901 2053 | 4975 2054 | 1930 2055 | 3690 2056 | 5358 2057 | 7049 2058 | 5047 2059 | 7135 2060 | 2745 2061 | 6031 2062 | 534 2063 | 10265 2064 | 8324 2065 | 579 2066 | 4335 2067 | 9856 2068 | 5912 2069 | 6108 2070 | 1852 2071 | 4732 2072 | 9934 2073 | 8006 2074 | 4639 2075 | 7004 2076 | 7959 2077 | 5846 2078 | 268 2079 | 5896 2080 | 3523 2081 | 441 2082 | 7803 2083 | 1268 2084 | 1154 2085 | 3136 2086 | 9658 2087 | 4162 2088 | 395 2089 | 3127 2090 | 2289 2091 | 3746 2092 | 7810 2093 | 6379 2094 | 3319 2095 | 9955 2096 | 648 2097 | 4581 2098 | 4549 2099 | 3118 2100 | 244 2101 | 6106 2102 | 10044 2103 | 4393 2104 | 8098 2105 | 7218 2106 | 2681 2107 | 7576 2108 | 4651 2109 | 7002 2110 | 8971 2111 | 3675 2112 | 5510 2113 | 2960 2114 | 2615 2115 | 7327 2116 | 8939 2117 | 6086 2118 | 4991 2119 | 2706 2120 | 1025 2121 | 1239 2122 | 1740 2123 | 7389 2124 | 3723 2125 | 6705 2126 | 658 2127 | 4801 2128 | 9016 2129 | 765 2130 | 5534 2131 | 1541 2132 | 2846 2133 | 4456 2134 | 9294 2135 | 166 2136 | 5410 2137 | 8450 2138 | 8326 2139 | 192 2140 | 4863 2141 | 399 2142 | 9574 2143 | 284 2144 | 3640 2145 | 3327 2146 | 1007 2147 | 1060 2148 | 3501 2149 | 7689 2150 | 6026 2151 | 6913 2152 | 5258 2153 | 5724 2154 | 3976 2155 | 8797 2156 | 5895 2157 | 4635 2158 | 10473 2159 | 3338 2160 | 2465 2161 | 8855 2162 | 6609 2163 | 10511 2164 | 3502 2165 | 2730 2166 | 8451 2167 | 1488 2168 | 1926 2169 | 5890 2170 | 9873 2171 | 2415 2172 | 5908 2173 | 3597 2174 | 3654 2175 | 9451 2176 | 3456 2177 | 9617 2178 | 5522 2179 | 8595 2180 | 6294 2181 | 7422 2182 | 10176 2183 | 1584 2184 | 5738 2185 | 2555 2186 | 5383 2187 | 6680 2188 | 5927 2189 | 1877 2190 | 6736 2191 | 6907 2192 | 4336 2193 | 4997 2194 | 8188 2195 | 3180 2196 | 662 2197 | 1997 2198 | 3040 2199 | 8784 2200 | 6476 2201 | 9889 2202 | 9989 2203 | 4648 2204 | 10510 2205 | 9609 2206 | 1961 2207 | 939 2208 | 3938 2209 | 6696 2210 | 2013 2211 | 6545 2212 | 6618 2213 | 4859 2214 | 842 2215 | 7479 2216 | 3957 2217 | 6308 2218 | 392 2219 | 8803 2220 | 4979 2221 | 6830 2222 | 5642 2223 | 10038 2224 | 10036 2225 | 8708 2226 | 1692 2227 | 5063 2228 | 8361 2229 | 2252 2230 | 4107 2231 | 584 2232 | 4957 2233 | 2459 2234 | 6946 2235 | 4531 2236 | 2272 2237 | 2335 2238 | 8436 2239 | 9143 2240 | 5566 2241 | 602 2242 | 5801 2243 | 5546 2244 | 5109 2245 | 3864 2246 | 6613 2247 | 2413 2248 | 9926 2249 | 916 2250 | 7871 2251 | 7390 2252 | 10294 2253 | 3735 2254 | 4060 2255 | 6291 2256 | 6007 2257 | 514 2258 | 4209 2259 | 8657 2260 | 8462 2261 | 2108 2262 | 6050 2263 | 4641 2264 | 6330 2265 | 5888 2266 | 5412 2267 | 8336 2268 | 1192 2269 | 9394 2270 | 6033 2271 | 276 2272 | 9317 2273 | 4770 2274 | 7823 2275 | 5660 2276 | 7256 2277 | 10027 2278 | 1814 2279 | 1439 2280 | 10015 2281 | 2896 2282 | 7345 2283 | 8958 2284 | 6096 2285 | 9790 2286 | 7979 2287 | 1344 2288 | 3113 2289 | 3323 2290 | 10622 2291 | 10523 2292 | 7516 2293 | 769 2294 | 3339 2295 | 9110 2296 | 9425 2297 | 7936 2298 | 2601 2299 | 9358 2300 | 7192 2301 | 10448 2302 | 10463 2303 | 1487 2304 | 5529 2305 | 4850 2306 | 5290 2307 | 1033 2308 | 9999 2309 | 2055 2310 | 1449 2311 | 2400 2312 | 7468 2313 | 10275 2314 | 9647 2315 | 5152 2316 | 5272 2317 | 5995 2318 | 1136 2319 | 9853 2320 | 2347 2321 | 9093 2322 | 5110 2323 | 8128 2324 | 8676 2325 | 7570 2326 | 3144 2327 | 9380 2328 | 4447 2329 | 6888 2330 | 6274 2331 | 1524 2332 | 4248 2333 | 10200 2334 | 6970 2335 | 601 2336 | 7030 2337 | 3213 2338 | 6868 2339 | 4721 2340 | 1721 2341 | 1640 2342 | 5340 2343 | 7967 2344 | 6565 2345 | 8157 2346 | 703 2347 | 10566 2348 | 8875 2349 | 8514 2350 | 1181 2351 | 4377 2352 | 4699 2353 | 4071 2354 | 24 2355 | 6894 2356 | 2222 2357 | 7415 2358 | 3935 2359 | 9460 2360 | 3797 2361 | 7879 2362 | 3198 2363 | 356 2364 | 7058 2365 | 6936 2366 | 332 2367 | 5712 2368 | 2824 2369 | 489 2370 | 7795 2371 | 10258 2372 | 1394 2373 | 4299 2374 | 9377 2375 | 4196 2376 | 1774 2377 | 1537 2378 | 7017 2379 | 9084 2380 | 6047 2381 | 8052 2382 | 10646 2383 | 1211 2384 | 623 2385 | 7413 2386 | 9164 2387 | 3909 2388 | 9641 2389 | 5020 2390 | 5619 2391 | 3488 2392 | 1228 2393 | 1457 2394 | 4784 2395 | 9907 2396 | 3800 2397 | 3454 2398 | 3369 2399 | 4824 2400 | 8736 2401 | 10184 2402 | 7107 2403 | 10206 2404 | 7114 2405 | 9054 2406 | 8467 2407 | 10253 2408 | 1422 2409 | 5093 2410 | 3420 2411 | 2047 2412 | 1368 2413 | 5424 2414 | 5111 2415 | 8163 2416 | 9258 2417 | 3646 2418 | 1242 2419 | 1777 2420 | 10387 2421 | 1261 2422 | 5471 2423 | 501 2424 | 9261 2425 | 7504 2426 | 2891 2427 | 7757 2428 | 9225 2429 | 3086 2430 | 9642 2431 | 1454 2432 | 9122 2433 | 6023 2434 | 7625 2435 | 3767 2436 | 5780 2437 | 2932 2438 | 9116 2439 | 10659 2440 | 379 2441 | 810 2442 | 369 2443 | 5541 2444 | 1418 2445 | 2548 2446 | 6333 2447 | 4262 2448 | 4115 2449 | 7994 2450 | 8346 2451 | 3160 2452 | 3360 2453 | 3256 2454 | 3337 2455 | 1023 2456 | 3359 2457 | 6588 2458 | 2749 2459 | 4858 2460 | 10602 2461 | 1297 2462 | 3081 2463 | 1172 2464 | 3099 2465 | 3413 2466 | 637 2467 | 3545 2468 | 784 2469 | 10172 2470 | 8206 2471 | 6217 2472 | 7768 2473 | 3204 2474 | 1347 2475 | 6251 2476 | 4161 2477 | 8232 2478 | 7047 2479 | 2203 2480 | 7094 2481 | 4111 2482 | 2709 2483 | 10438 2484 | 9278 2485 | 5160 2486 | 936 2487 | 10616 2488 | 9418 2489 | 249 2490 | 1251 2491 | 8292 2492 | 9262 2493 | 3036 2494 | 6651 2495 | 3103 2496 | 8239 2497 | 1415 2498 | 4166 2499 | 3128 2500 | 760 2501 | 1739 2502 | 6186 2503 | 8061 2504 | 4016 2505 | 8287 2506 | 6155 2507 | 7475 2508 | 5315 2509 | 7657 2510 | 1325 2511 | 10606 2512 | 5791 2513 | 1868 2514 | 10092 2515 | 5980 2516 | 3356 2517 | 2770 2518 | 5992 2519 | 2424 2520 | 10059 2521 | 800 2522 | 8897 2523 | 1908 2524 | 3148 2525 | 9710 2526 | 3661 2527 | 6917 2528 | 169 2529 | 6365 2530 | 1526 2531 | 5567 2532 | 7073 2533 | 3257 2534 | 10284 2535 | 9212 2536 | 5582 2537 | 2304 2538 | 4533 2539 | 5761 2540 | 10197 2541 | 2731 2542 | 5592 2543 | 9323 2544 | 2394 2545 | 2938 2546 | 6949 2547 | 8389 2548 | 4613 2549 | 574 2550 | 615 2551 | 1659 2552 | 7637 2553 | 10343 2554 | 10403 2555 | 7444 2556 | 6052 2557 | 4066 2558 | 5249 2559 | 2714 2560 | 2833 2561 | 3627 2562 | 7446 2563 | 2782 2564 | 3000 2565 | 8744 2566 | 2677 2567 | 10420 2568 | 1426 2569 | 6442 2570 | 4568 2571 | 9542 2572 | 1607 2573 | 8400 2574 | 4781 2575 | 3540 2576 | 8817 2577 | 2069 2578 | 1316 2579 | 109 2580 | 2635 2581 | 3989 2582 | 202 2583 | 8830 2584 | 1209 2585 | 3161 2586 | 6591 2587 | 9259 2588 | 7532 2589 | 1719 2590 | 9218 2591 | 10250 2592 | 4053 2593 | 2778 2594 | 4851 2595 | 8633 2596 | 4075 2597 | 5985 2598 | 10075 2599 | 9005 2600 | 8550 2601 | 9027 2602 | 37 2603 | 4738 2604 | 131 2605 | 1593 2606 | 4839 2607 | 2011 2608 | 4 2609 | 9047 2610 | 1911 2611 | 679 2612 | 7696 2613 | 5521 2614 | 397 2615 | 1648 2616 | 3002 2617 | 460 2618 | 2565 2619 | 10421 2620 | 7507 2621 | 9608 2622 | 2536 2623 | 5875 2624 | 7546 2625 | 8721 2626 | 6682 2627 | 5508 2628 | 121 2629 | 5414 2630 | 4614 2631 | 1057 2632 | 9722 2633 | 1005 2634 | 7033 2635 | 3998 2636 | 3464 2637 | 1117 2638 | 4429 2639 | 4481 2640 | 7310 2641 | 5265 2642 | 1799 2643 | 4470 2644 | 5257 2645 | 1470 2646 | 7515 2647 | 9282 2648 | 6679 2649 | 1103 2650 | 8075 2651 | 9315 2652 | 10353 2653 | 7533 2654 | 1219 2655 | 2786 2656 | 9483 2657 | 3816 2658 | 5940 2659 | 8095 2660 | 2661 2661 | 7585 2662 | 9179 2663 | 1920 2664 | 3306 2665 | 6847 2666 | 9252 2667 | 5600 2668 | 8420 2669 | 4195 2670 | 7744 2671 | 7641 2672 | 7069 2673 | 5795 2674 | 7487 2675 | 2238 2676 | 4120 2677 | 2379 2678 | 112 2679 | 4270 2680 | 6721 2681 | 5361 2682 | 9527 2683 | 10629 2684 | 9502 2685 | 1865 2686 | 6984 2687 | 10575 2688 | 2144 2689 | 2243 2690 | 600 2691 | 3794 2692 | 8954 2693 | 9805 2694 | 4729 2695 | 4273 2696 | 1756 2697 | 4050 2698 | 19 2699 | 5206 2700 | 6071 2701 | 1065 2702 | 2822 2703 | 9811 2704 | 530 2705 | 7103 2706 | 9815 2707 | 1714 2708 | 8285 2709 | 2784 2710 | 3806 2711 | 10472 2712 | 9404 2713 | 10583 2714 | 7136 2715 | 8184 2716 | 6957 2717 | 5115 2718 | 4959 2719 | 4951 2720 | 3255 2721 | 167 2722 | 8113 2723 | 5193 2724 | 235 2725 | 1896 2726 | 3293 2727 | 3243 2728 | 6769 2729 | 6971 2730 | 10581 2731 | 2082 2732 | 8323 2733 | 5802 2734 | 7351 2735 | 142 2736 | 1401 2737 | 3430 2738 | 1862 2739 | 8570 2740 | 10078 2741 | 1441 2742 | 74 2743 | 543 2744 | 8730 2745 | 975 2746 | 5198 2747 | 9522 2748 | 5538 2749 | 7525 2750 | 4936 2751 | 8372 2752 | 7175 2753 | 5881 2754 | 5081 2755 | 8118 2756 | 2660 2757 | 10047 2758 | 9639 2759 | 5962 2760 | 1560 2761 | 175 2762 | 8759 2763 | 9336 2764 | 2385 2765 | 5788 2766 | 5235 2767 | 8226 2768 | 2097 2769 | 1687 2770 | 1098 2771 | 10364 2772 | 2916 2773 | 823 2774 | 8685 2775 | 9272 2776 | 847 2777 | 6849 2778 | 5889 2779 | 6660 2780 | 10003 2781 | 5366 2782 | 7277 2783 | 4280 2784 | 7338 2785 | 5574 2786 | 3570 2787 | 4187 2788 | 5809 2789 | 3996 2790 | 7377 2791 | 8219 2792 | 6908 2793 | 10357 2794 | 10390 2795 | 294 2796 | 3793 2797 | 3447 2798 | 6502 2799 | 5689 2800 | 3142 2801 | 210 2802 | 544 2803 | 7442 2804 | 9649 2805 | 1765 2806 | 7713 2807 | 6055 2808 | 7019 2809 | 1954 2810 | 828 2811 | 3563 2812 | 4892 2813 | 2688 2814 | 7438 2815 | 3543 2816 | 509 2817 | 7176 2818 | 8619 2819 | 7149 2820 | 538 2821 | 6562 2822 | 6777 2823 | 3037 2824 | 10388 2825 | 3117 2826 | 8892 2827 | 8845 2828 | 2858 2829 | 2746 2830 | 1905 2831 | 5989 2832 | 5747 2833 | 9310 2834 | 9154 2835 | 10329 2836 | 6690 2837 | 1895 2838 | 1069 2839 | 5919 2840 | 3461 2841 | 6713 2842 | 9956 2843 | 6615 2844 | 8608 2845 | 808 2846 | 6287 2847 | 8695 2848 | 5443 2849 | 4378 2850 | 2758 2851 | 2003 2852 | 8205 2853 | 5513 2854 | 1353 2855 | 8694 2856 | 6606 2857 | 2764 2858 | 2315 2859 | 5851 2860 | 9471 2861 | 1595 2862 | 6056 2863 | 6316 2864 | 9626 2865 | 933 2866 | 396 2867 | 1800 2868 | 10201 2869 | 1709 2870 | 4241 2871 | 8751 2872 | 9559 2873 | 5769 2874 | 6666 2875 | 1539 2876 | 4453 2877 | 2182 2878 | 3892 2879 | 5681 2880 | 8412 2881 | 2607 2882 | 3228 2883 | 1672 2884 | 2134 2885 | 6590 2886 | 8271 2887 | 3114 2888 | 2558 2889 | 99 2890 | 2080 2891 | 6912 2892 | 4841 2893 | 4793 2894 | 4583 2895 | 2440 2896 | 1809 2897 | 5730 2898 | 2398 2899 | 7827 2900 | 5552 2901 | 8500 2902 | 3631 2903 | 5563 2904 | 8477 2905 | 2426 2906 | 2102 2907 | 9003 2908 | 4796 2909 | 6597 2910 | 3334 2911 | 7931 2912 | 3730 2913 | 9659 2914 | 9946 2915 | 6862 2916 | 3760 2917 | 9490 2918 | 3838 2919 | 2225 2920 | 480 2921 | 3785 2922 | 2454 2923 | 4101 2924 | 6403 2925 | 9625 2926 | 4697 2927 | 2574 2928 | 1918 2929 | 9117 2930 | 7041 2931 | 7122 2932 | 6065 2933 | 5647 2934 | 1240 2935 | 2884 2936 | 865 2937 | 6593 2938 | 5905 2939 | 5745 2940 | 1876 2941 | 7690 2942 | 2819 2943 | 5080 2944 | 125 2945 | 7284 2946 | 9361 2947 | 2253 2948 | 1960 2949 | 9980 2950 | 8943 2951 | 10625 2952 | 4884 2953 | 6084 2954 | 4789 2955 | 10370 2956 | 9867 2957 | 6199 2958 | 1462 2959 | 3195 2960 | 9966 2961 | 4386 2962 | 3088 2963 | 10051 2964 | 982 2965 | 6564 2966 | 5978 2967 | 5102 2968 | 7043 2969 | 9618 2970 | 8921 2971 | 6526 2972 | 3827 2973 | 6470 2974 | 5421 2975 | 2214 2976 | 8126 2977 | 6831 2978 | 8778 2979 | 2299 2980 | 9292 2981 | 4021 2982 | 10 2983 | 5632 2984 | 4902 2985 | 7272 2986 | 2239 2987 | 9429 2988 | 1055 2989 | 7113 2990 | 6942 2991 | 4385 2992 | 9450 2993 | 5612 2994 | 4616 2995 | 310 2996 | 1299 2997 | 6232 2998 | 9255 2999 | 9567 3000 | 8192 3001 | 6205 3002 | 1963 3003 | 4760 3004 | 5430 3005 | 3038 3006 | 9170 3007 | 3064 3008 | 3145 3009 | 7087 3010 | 343 3011 | 5384 3012 | 6262 3013 | 4980 3014 | 10429 3015 | 165 3016 | 10146 3017 | 783 3018 | 1934 3019 | 7208 3020 | 8886 3021 | 8911 3022 | 6781 3023 | 1013 3024 | 3102 3025 | 5246 3026 | 5657 3027 | 10346 3028 | 4370 3029 | 8083 3030 | 9018 3031 | 6331 3032 | 9432 3033 | 7074 3034 | 6948 3035 | 5832 3036 | 2594 3037 | 7258 3038 | 8956 3039 | 8555 3040 | 7522 3041 | 2156 3042 | 2870 3043 | 10194 3044 | 3590 3045 | 8049 3046 | 3208 3047 | 4909 3048 | 805 3049 | 223 3050 | 5274 3051 | 10435 3052 | 5859 3053 | 5611 3054 | 9301 3055 | 8931 3056 | 1962 3057 | 8622 3058 | 3828 3059 | 2320 3060 | 2994 3061 | 4856 3062 | 6587 3063 | 860 3064 | 1798 3065 | 5346 3066 | 4650 3067 | 10615 3068 | 6124 3069 | 5516 3070 | 9060 3071 | 4972 3072 | 9731 3073 | 2202 3074 | 4561 3075 | 2604 3076 | 1802 3077 | 7268 3078 | 4381 3079 | 8568 3080 | 8912 3081 | 8629 3082 | 8444 3083 | 3006 3084 | 3862 3085 | 4624 3086 | 2476 3087 | 9311 3088 | 10007 3089 | 10117 3090 | 8259 3091 | 6980 3092 | 8867 3093 | 10103 3094 | 8162 3095 | 2816 3096 | 2729 3097 | 4983 3098 | 3378 3099 | 2926 3100 | 8138 3101 | 9267 3102 | 547 3103 | 4827 3104 | 4908 3105 | 6771 3106 | 2793 3107 | 852 3108 | 555 3109 | 8248 3110 | 447 3111 | 6981 3112 | 3363 3113 | 7096 3114 | 10233 3115 | 5052 3116 | 4672 3117 | 4106 3118 | 6466 3119 | 6264 3120 | 2616 3121 | 7863 3122 | 7296 3123 | 4424 3124 | 3664 3125 | 10048 3126 | 9401 3127 | 8833 3128 | 3469 3129 | 8593 3130 | 4513 3131 | 9668 3132 | 2298 3133 | 6723 3134 | 4566 3135 | 816 3136 | 5325 3137 | 8546 3138 | 3207 3139 | 9036 3140 | 5293 3141 | 3226 3142 | 4036 3143 | 2149 3144 | 4024 3145 | 4682 3146 | 257 3147 | 9067 3148 | 10096 3149 | 9297 3150 | 6254 3151 | 3557 3152 | 6655 3153 | 7067 3154 | 10547 3155 | 7781 3156 | 8267 3157 | 2811 3158 | 8930 3159 | 7145 3160 | 2689 3161 | 9537 3162 | 3326 3163 | 9916 3164 | 9360 3165 | 6910 3166 | 7016 3167 | 10060 3168 | 9937 3169 | 2444 3170 | 10474 3171 | 8990 3172 | 2301 3173 | 10064 3174 | 8696 3175 | 4539 3176 | 7085 3177 | 4154 3178 | 159 3179 | 7381 3180 | 3230 3181 | 5494 3182 | 8876 3183 | 2268 3184 | 5371 3185 | 6746 3186 | 10555 3187 | 7927 3188 | 9402 3189 | 591 3190 | 10458 3191 | 7786 3192 | 3158 3193 | 10372 3194 | 5827 3195 | 3867 3196 | 2213 3197 | 5466 3198 | 3673 3199 | 4599 3200 | 1699 3201 | 323 3202 | 1880 3203 | 10189 3204 | 4747 3205 | 8291 3206 | 7216 3207 | 9409 3208 | 8621 3209 | 9019 3210 | 3840 3211 | 4723 3212 | 2099 3213 | 5988 3214 | 8311 3215 | 6694 3216 | 7614 3217 | 8521 3218 | 4887 3219 | 1888 3220 | 3595 3221 | 5711 3222 | 8129 3223 | 8270 3224 | 5474 3225 | 9484 3226 | 6573 3227 | 4722 3228 | 2141 3229 | 8638 3230 | 6529 3231 | 129 3232 | 7385 3233 | 6520 3234 | 5292 3235 | 2608 3236 | 3761 3237 | 2540 3238 | 6311 3239 | 4434 3240 | 5690 3241 | 2618 3242 | 8970 3243 | 9286 3244 | 1340 3245 | 8689 3246 | 7674 3247 | 10363 3248 | 5964 3249 | 7999 3250 | 1332 3251 | 1445 3252 | 1568 3253 | 152 3254 | 9570 3255 | 6608 3256 | 8322 3257 | 8491 3258 | 1263 3259 | 10588 3260 | 4445 3261 | 7841 3262 | 2015 3263 | 9048 3264 | 10409 3265 | 7040 3266 | 9701 3267 | 6104 3268 | 1485 3269 | 7887 3270 | 128 3271 | 9724 3272 | 8908 3273 | 2412 3274 | 8711 3275 | 4216 3276 | 4792 3277 | 4922 3278 | 3181 3279 | 8718 3280 | 3525 3281 | 4687 3282 | 10063 3283 | 3745 3284 | 872 3285 | 1453 3286 | 4604 3287 | 4129 3288 | 4817 3289 | 4519 3290 | 3439 3291 | 10560 3292 | 8812 3293 | 4593 3294 | 3530 3295 | 9911 3296 | 2921 3297 | 3682 3298 | 22 3299 | 1746 3300 | 3834 3301 | 528 3302 | 6931 3303 | 1819 3304 | 9288 3305 | 8587 3306 | 5581 3307 | 3071 3308 | 7523 3309 | 744 3310 | 6678 3311 | 962 3312 | 1176 3313 | 4819 3314 | 9343 3315 | 3911 3316 | 1790 3317 | 7890 3318 | 6700 3319 | 9002 3320 | 2592 3321 | 9124 3322 | 6362 3323 | 1247 3324 | 6020 3325 | 9509 3326 | 8925 3327 | 9249 3328 | 7531 3329 | 8775 3330 | 5144 3331 | 4200 3332 | 1836 3333 | 6003 3334 | 736 3335 | 10118 3336 | 6474 3337 | 8277 3338 | 9983 3339 | 4714 3340 | 3377 3341 | 9442 3342 | 6790 3343 | 5637 3344 | 3829 3345 | 42 3346 | 2433 3347 | 6726 3348 | 9279 3349 | 8139 3350 | 10006 3351 | 7738 3352 | 6711 3353 | 2946 3354 | 1576 3355 | 2740 3356 | 8485 3357 | 2814 3358 | 619 3359 | 9943 3360 | 7550 3361 | 1180 3362 | 4350 3363 | 384 3364 | 3644 3365 | 715 3366 | 7404 3367 | 141 3368 | 3922 3369 | 5142 3370 | 5697 3371 | 2576 3372 | 2290 3373 | 7597 3374 | 3320 3375 | 170 3376 | 3129 3377 | 5225 3378 | 7675 3379 | 7093 3380 | 4353 3381 | 5029 3382 | 1227 3383 | 7774 3384 | 5330 3385 | 10268 3386 | 3505 3387 | 10062 3388 | 9187 3389 | 5525 3390 | 888 3391 | 9367 3392 | 6424 3393 | 977 3394 | 8765 3395 | 7064 3396 | 2537 3397 | 2453 3398 | 3705 3399 | 10304 3400 | 926 3401 | 793 3402 | 4875 3403 | 704 3404 | 8301 3405 | 9448 3406 | 8927 3407 | 57 3408 | 6735 3409 | 7118 3410 | 3587 3411 | 3247 3412 | 10105 3413 | 5606 3414 | 9779 3415 | 6323 3416 | 3905 3417 | 114 3418 | 5046 3419 | 3753 3420 | 9802 3421 | 3396 3422 | 3341 3423 | 5994 3424 | 9807 3425 | 3945 3426 | 1387 3427 | 8767 3428 | 10309 3429 | 7086 3430 | 7234 3431 | 8308 3432 | 7369 3433 | 6420 3434 | 8302 3435 | 9211 3436 | 4329 3437 | 6256 3438 | 4544 3439 | 9481 3440 | 2027 3441 | 8123 3442 | 8869 3443 | 3861 3444 | 2305 3445 | 8479 3446 | 628 3447 | 2477 3448 | 573 3449 | 6336 3450 | 4086 3451 | 214 3452 | 5813 3453 | 6595 3454 | 8567 3455 | 1603 3456 | 10152 3457 | 4407 3458 | 9808 3459 | 2270 3460 | 1665 3461 | 5306 3462 | 1928 3463 | 4159 3464 | 10489 3465 | 9995 3466 | 8227 3467 | 2331 3468 | 7171 3469 | 6122 3470 | 3854 3471 | 4191 3472 | 6695 3473 | 7843 3474 | 4878 3475 | 1288 3476 | 10396 3477 | 7264 3478 | 8967 3479 | 1319 3480 | 6972 3481 | 2687 3482 | 2468 3483 | 6198 3484 | 2589 3485 | 5168 3486 | 2407 3487 | 5701 3488 | 10029 3489 | 9857 3490 | 3913 3491 | 8202 3492 | 4091 3493 | 9598 3494 | 3506 3495 | 9918 3496 | 7154 3497 | 7165 3498 | 6863 3499 | 4890 3500 | 5048 3501 | 1017 3502 | 6393 3503 | 4462 3504 | 8288 3505 | 6489 3506 | 4832 3507 | 7211 3508 | 8704 3509 | 7662 3510 | 9594 3511 | 10601 3512 | 7521 3513 | 10411 3514 | 2006 3515 | 2988 3516 | 4690 3517 | 6828 3518 | 8777 3519 | 7779 3520 | 3920 3521 | 1331 3522 | 3750 3523 | 10106 3524 | 2539 3525 | 8339 3526 | 5931 3527 | 9241 3528 | 5613 3529 | 1967 3530 | 5438 3531 | 6953 3532 | 10519 3533 | 5060 3534 | 1727 3535 | 2517 3536 | 9733 3537 | 9888 3538 | 2430 3539 | 6843 3540 | 9058 3541 | 9376 3542 | 6272 3543 | 10129 3544 | 8041 3545 | 376 3546 | 1645 3547 | 694 3548 | 9903 3549 | 4167 3550 | 5727 3551 | 7557 3552 | 3421 3553 | 8613 3554 | 8137 3555 | 3915 3556 | 4946 3557 | 10157 3558 | 8630 3559 | 1737 3560 | 3713 3561 | 5321 3562 | 7408 3563 | 2792 3564 | 7527 3565 | 8681 3566 | 8440 3567 | 1321 3568 | 9610 3569 | 4708 3570 | 8496 3571 | 665 3572 | 9785 3573 | 5147 3574 | 868 3575 | 9750 3576 | 7891 3577 | 2895 3578 | 5511 3579 | 774 3580 | 9102 3581 | 5082 3582 | 5523 3583 | 1218 3584 | 6673 3585 | 1818 3586 | 10241 3587 | 6417 3588 | 3132 3589 | 8251 3590 | 10570 3591 | 3311 3592 | 8915 3593 | 120 3594 | 15 3595 | 9004 3596 | 10537 3597 | 354 3598 | 8149 3599 | 5570 3600 | 4743 3601 | 3431 3602 | 3121 3603 | 9611 3604 | 68 3605 | 1992 3606 | 7402 3607 | 4522 3608 | 617 3609 | 130 3610 | 5280 3611 | 1215 3612 | 97 3613 | 371 3614 | 6118 3615 | 10493 3616 | 3990 3617 | 1907 3618 | 6028 3619 | 3766 3620 | 3689 3621 | 7868 3622 | 7968 3623 | 5967 3624 | 2048 3625 | 86 3626 | 9075 3627 | 5793 3628 | 5334 3629 | 3362 3630 | 6551 3631 | 9037 3632 | 53 3633 | 5656 3634 | 1701 3635 | 8210 3636 | 3302 3637 | 5236 3638 | 1271 3639 | 1420 3640 | 6245 3641 | 10498 3642 | 3178 3643 | 7672 3644 | 7807 3645 | 5588 3646 | 8260 3647 | 5997 3648 | 8161 3649 | 2210 3650 | 4607 3651 | 2139 3652 | 3372 3653 | 5852 3654 | 6284 3655 | 5654 3656 | 3405 3657 | 1845 3658 | 492 3659 | 6999 3660 | 4428 3661 | 3707 3662 | 588 3663 | 3965 3664 | 10222 3665 | 4715 3666 | 6343 3667 | 10065 3668 | 8443 3669 | 8017 3670 | 8478 3671 | 2868 3672 | 1782 3673 | 5196 3674 | 1287 3675 | 1147 3676 | 10155 3677 | 2708 3678 | 1492 3679 | 4589 3680 | 6044 3681 | 6426 3682 | 889 3683 | 5706 3684 | 6150 3685 | 9838 3686 | 4916 3687 | 8879 3688 | 179 3689 | 3880 3690 | 4621 3691 | 4097 3692 | 4719 3693 | 1298 3694 | 8004 3695 | 1711 3696 | 10249 3697 | 9664 3698 | 8181 3699 | 2132 3700 | 5866 3701 | 1403 3702 | 4324 3703 | 5465 3704 | 1293 3705 | 3553 3706 | 7835 3707 | 7615 3708 | 2244 3709 | 7536 3710 | 4866 3711 | 1234 3712 | 6090 3713 | 2226 3714 | 5100 3715 | 9760 3716 | 7412 3717 | 8418 3718 | 5737 3719 | 2849 3720 | 9685 3721 | 1898 3722 | 3349 3723 | 8558 3724 | 3809 3725 | 2595 3726 | 6546 3727 | 9508 3728 | 3153 3729 | 1811 3730 | 7259 3731 | 10597 3732 | 5341 3733 | 2528 3734 | 10070 3735 | 2323 3736 | 8668 3737 | 7241 3738 | 10219 3739 | 1697 3740 | 5137 3741 | 9422 3742 | 10067 3743 | 7717 3744 | 4170 3745 | 5626 3746 | 3301 3747 | 2909 3748 | 8024 3749 | 5335 3750 | 4027 3751 | 9072 3752 | 675 3753 | 8776 3754 | 3392 3755 | 7755 3756 | 4051 3757 | 3315 3758 | 9167 3759 | 3425 3760 | 5252 3761 | 6073 3762 | 5473 3763 | 6704 3764 | 116 3765 | 2796 3766 | 7865 3767 | 890 3768 | 10178 3769 | 7060 3770 | 1715 3771 | 6522 3772 | 8472 3773 | 444 3774 | 2393 3775 | 2138 3776 | 825 3777 | 6249 3778 | 5283 3779 | 10239 3780 | 2719 3781 | 4567 3782 | 2622 3783 | 1731 3784 | 6638 3785 | 747 3786 | 4485 3787 | 5857 3788 | 373 3789 | 3284 3790 | 10432 3791 | 1921 3792 | 2603 3793 | 9950 3794 | 5702 3795 | 4281 3796 | 4061 3797 | 8994 3798 | 6260 3799 | 10584 3800 | 5380 3801 | 10431 3802 | 7110 3803 | 7388 3804 | 1222 3805 | 2564 3806 | 978 3807 | 7867 3808 | 795 3809 | 8686 3810 | 8974 3811 | 8176 3812 | 9100 3813 | 1150 3814 | 7790 3815 | 4029 3816 | 5074 3817 | 7669 3818 | 8781 3819 | 2359 3820 | 2542 3821 | 1309 3822 | 2441 3823 | 1270 3824 | 1555 3825 | 4408 3826 | 3034 3827 | 9578 3828 | 4063 3829 | 10056 3830 | 1831 3831 | 3019 3832 | 8016 3833 | 8789 3834 | 9424 3835 | 921 3836 | 9330 3837 | 3879 3838 | 1912 3839 | 7130 3840 | 9213 3841 | 8916 3842 | 1776 3843 | 1388 3844 | 3598 3845 | 3047 3846 | 9443 3847 | 1591 3848 | 3111 3849 | 3704 3850 | 234 3851 | 7396 3852 | 6856 3853 | 1635 3854 | 4791 3855 | 883 3856 | 3028 3857 | 1864 3858 | 3001 3859 | 6246 3860 | 1021 3861 | 9506 3862 | 864 3863 | 2370 3864 | 3537 3865 | 7231 3866 | 6250 3867 | 737 3868 | 3168 3869 | 6837 3870 | 5175 3871 | 7966 3872 | 8115 3873 | 5307 3874 | 9081 3875 | 1770 3876 | 10010 3877 | 9777 3878 | 8172 3879 | 317 3880 | 9718 3881 | 8469 3882 | 4637 3883 | 7493 3884 | 2471 3885 | 785 3886 | 2036 3887 | 8743 3888 | 3850 3889 | 10441 3890 | 7771 3891 | 5806 3892 | 3780 3893 | 4318 3894 | 3370 3895 | 5289 3896 | 6927 3897 | 9730 3898 | 1452 3899 | 3027 3900 | 7652 3901 | 5803 3902 | 5391 3903 | 6427 3904 | 4731 3905 | 8337 3906 | 6172 3907 | 3655 3908 | 8076 3909 | 9049 3910 | 4395 3911 | 101 3912 | 1366 3913 | 7466 3914 | 3649 3915 | 5437 3916 | 3982 3917 | 8439 3918 | 9043 3919 | 9595 3920 | 2847 3921 | 4045 3922 | 1290 3923 | 3605 3924 | 6299 3925 | 1018 3926 | 9007 3927 | 1626 3928 | 7862 3929 | 8353 3930 | 8722 3931 | 465 3932 | 2212 3933 | 6675 3934 | 3856 3935 | 5495 3936 | 7116 3937 | 7686 3938 | 4038 3939 | 6548 3940 | 4354 3941 | 9503 3942 | 10089 3943 | 2300 3944 | 6438 3945 | 8842 3946 | 9914 3947 | 5007 3948 | 4452 3949 | 6715 3950 | 5880 3951 | 7115 3952 | 944 3953 | 3825 3954 | 3353 3955 | 5561 3956 | 4985 3957 | 9746 3958 | 1656 3959 | 3549 3960 | 10119 3961 | 2641 3962 | 2497 3963 | 8814 3964 | 7737 3965 | 10099 3966 | 6505 3967 | 9855 3968 | 2898 3969 | 4586 3970 | 5448 3971 | 5195 3972 | 5853 3973 | 8112 3974 | 727 3975 | 1461 3976 | 1034 3977 | 8998 3978 | 7872 3979 | 462 3980 | 689 3981 | 7297 3982 | 7108 3983 | 9233 3984 | 1546 3985 | 2546 3986 | 8825 3987 | 7496 3988 | 7281 3989 | 2993 3990 | 6503 3991 | 10633 3992 | 10149 3993 | 8333 3994 | 691 3995 | 3196 3996 | 2435 3997 | 4062 3998 | 7276 3999 | 9972 4000 | 9420 4001 | 2380 4002 | 1345 4003 | 3475 4004 | 4912 4005 | 9706 4006 | 8386 4007 | 6787 4008 | 1072 4009 | 6101 4010 | 195 4011 | 6623 4012 | 4279 4013 | 5615 4014 | 9066 4015 | 3305 4016 | 248 4017 | 2372 4018 | 1258 4019 | 10515 4020 | 9897 4021 | 2 4022 | 8944 4023 | 2296 4024 | 6335 4025 | 3960 4026 | 7789 4027 | 461 4028 | 9357 4029 | 723 4030 | 3312 4031 | 1705 4032 | 4517 4033 | 6557 4034 | 6501 4035 | 4617 4036 | 8850 4037 | 2698 4038 | 2659 4039 | 4615 4040 | 4947 4041 | 7140 4042 | 6959 4043 | 3318 4044 | 10057 4045 | 5646 4046 | 9038 4047 | 7818 4048 | 2790 4049 | 1533 4050 | 406 4051 | 9125 4052 | 8258 4053 | 1752 4054 | 7949 4055 | 3067 4056 | 6788 4057 | 3969 4058 | 7219 4059 | 688 4060 | 10091 4061 | 2319 4062 | 3167 4063 | 10636 4064 | 4906 4065 | 4307 4066 | 7394 4067 | 4871 4068 | 7270 4069 | 6024 4070 | 8991 4071 | 3174 4072 | 8228 4073 | 10408 4074 | 9740 4075 | 2596 4076 | 5453 4077 | 5776 4078 | 2768 4079 | 9666 4080 | 4296 4081 | 2935 4082 | 5665 4083 | 3336 4084 | 4272 4085 | 8674 4086 | 8155 4087 | 722 4088 | 402 4089 | 6277 4090 | 8209 4091 | 9157 4092 | 4995 4093 | 4777 4094 | 8482 4095 | 4490 4096 | 9682 4097 | 9353 4098 | 3852 4099 | 8760 4100 | 250 4101 | 3923 4102 | 9619 4103 | 394 4104 | 8314 4105 | 1301 4106 | 5734 4107 | 8319 4108 | 10034 4109 | 5970 4110 | 9655 4111 | 1248 4112 | 4893 4113 | 1757 4114 | 10445 4115 | 7902 4116 | 4944 4117 | 3536 4118 | 9908 4119 | 8835 4120 | 5008 4121 | 5390 4122 | 437 4123 | 5514 4124 | 877 4125 | 4229 4126 | 1859 4127 | 7138 4128 | 10516 4129 | 8536 4130 | 2232 4131 | 4431 4132 | 9095 4133 | 7306 4134 | 10614 4135 | 10627 4136 | 6496 4137 | 2321 4138 | 7693 4139 | 322 4140 | 5234 4141 | 6833 4142 | 1785 4143 | 9646 4144 | 7541 4145 | 1663 4146 | 0 4147 | 683 4148 | 4173 4149 | 1598 4150 | 2314 4151 | 4486 4152 | 6933 4153 | 3183 4154 | 6882 4155 | 4064 4156 | 6145 4157 | 6187 4158 | 10406 4159 | 5016 4160 | 1511 4161 | 4274 4162 | 3157 4163 | 3112 4164 | 4576 4165 | 161 4166 | 2806 4167 | 2302 4168 | 8356 4169 | 1178 4170 | 3159 4171 | 6994 4172 | 533 4173 | 1328 4174 | 512 4175 | 6712 4176 | 6724 4177 | 4584 4178 | 2984 4179 | 2588 4180 | 10557 4181 | 10454 4182 | 7612 4183 | 6015 4184 | 3388 4185 | 2974 4186 | 8604 4187 | 10171 4188 | 3492 4189 | 6206 4190 | 8645 4191 | 8540 4192 | 10376 4193 | 8768 4194 | 4773 4195 | 7852 4196 | 9339 4197 | 2789 4198 | 8893 4199 | 10123 4200 | 1338 4201 | 540 4202 | 331 4203 | 3166 4204 | 9545 4205 | 188 4206 | 9022 4207 | 10392 4208 | 3778 4209 | 1221 4210 | 8272 4211 | 6134 4212 | 3639 4213 | 2961 4214 | 2330 4215 | 5431 4216 | 1565 4217 | 8556 4218 | 8250 4219 | 6282 4220 | 2428 4221 | 5682 4222 | 6739 4223 | 2216 4224 | 2208 4225 | 9716 4226 | 164 4227 | 3240 4228 | 10567 4229 | 10216 4230 | 7039 4231 | 5294 4232 | 856 4233 | 838 4234 | 6279 4235 | 468 4236 | 9773 4237 | 4502 4238 | 4190 4239 | 7877 4240 | 5924 4241 | 9513 4242 | 4618 4243 | 3812 4244 | 3089 4245 | 7105 4246 | 3432 4247 | 9809 4248 | 3970 4249 | 5668 4250 | 7530 4251 | 1681 4252 | 9529 4253 | 108 4254 | 364 4255 | 29 4256 | 739 4257 | 9872 4258 | 2853 4259 | 5188 4260 | 3939 4261 | 3210 4262 | 3529 4263 | 4356 4264 | 6836 4265 | 1260 4266 | 5493 4267 | 419 4268 | 2135 4269 | 139 4270 | 6916 4271 | 5097 4272 | 6034 4273 | 3052 4274 | 2478 4275 | 10127 4276 | 8995 4277 | 832 4278 | 5374 4279 | 8702 4280 | 3046 4281 | 10440 4282 | 2732 4283 | 3773 4284 | 2395 4285 | 6944 4286 | 6422 4287 | 9307 4288 | 5141 4289 | 3403 4290 | 2972 4291 | 10653 4292 | 7420 4293 | 8145 4294 | 181 4295 | 510 4296 | 9864 4297 | 10226 4298 | 2197 4299 | 7834 4300 | 115 4301 | 6458 4302 | 2037 4303 | 5123 4304 | 8701 4305 | 5622 4306 | 7605 4307 | 10604 4308 | 8421 4309 | 8177 4310 | 2164 4311 | 4172 4312 | 757 4313 | 9494 4314 | 2129 4315 | 3433 4316 | 7090 4317 | 5119 4318 | 4163 4319 | 4982 4320 | 1269 4321 | 9196 4322 | 2481 4323 | 2075 4324 | 3749 4325 | 814 4326 | 10469 4327 | 976 4328 | 6734 4329 | 7068 4330 | 2429 4331 | 4693 4332 | 10623 4333 | 6961 4334 | 2397 4335 | 8108 4336 | 6303 4337 | 6920 4338 | 2241 4339 | 6177 4340 | 4371 4341 | 2924 4342 | 2531 4343 | 9327 4344 | 3060 4345 | 9386 4346 | 7431 4347 | 289 4348 | 7845 4349 | 8182 4350 | 94 4351 | 8658 4352 | 4967 4353 | 4201 4354 | 9298 4355 | 833 4356 | 7535 4357 | 8362 4358 | 9707 4359 | 1909 4360 | 1498 4361 | 618 4362 | 3394 4363 | 1427 4364 | 8535 4365 | 4471 4366 | 132 4367 | 4224 4368 | 9169 4369 | 110 4370 | 4098 4371 | 6151 4372 | 3564 4373 | 4998 4374 | 6780 4375 | 4246 4376 | 9325 4377 | 9333 4378 | 9931 4379 | 790 4380 | 5913 4381 | 6600 4382 | 1349 4383 | 1040 4384 | 1135 4385 | 1400 4386 | 8442 4387 | 8223 4388 | 724 4389 | 87 4390 | 1482 4391 | 8526 4392 | 1718 4393 | 2431 4394 | 5264 4395 | 9852 4396 | 9971 4397 | 3491 4398 | 6875 4399 | 5464 4400 | 7839 4401 | 6119 4402 | 7621 4403 | 7321 4404 | 2162 4405 | 8938 4406 | 8717 4407 | 8381 4408 | 6057 4409 | 4044 4410 | 487 4411 | 4826 4412 | 1336 4413 | 2076 4414 | 3789 4415 | 9398 4416 | 6300 4417 | 10030 4418 | 5442 4419 | 3932 4420 | 3684 4421 | 2005 4422 | 5943 4423 | 8281 4424 | 5743 4425 | 9144 4426 | 1600 4427 | 5714 4428 | 8156 4429 | 9901 4430 | 3055 4431 | 3321 4432 | 4673 4433 | 4588 4434 | 7780 4435 | 9929 4436 | 8060 4437 | 10257 4438 | 7700 4439 | 1011 4440 | 7640 4441 | 3177 4442 | 6380 4443 | 8523 4444 | 2109 4445 | 3065 4446 | 7050 4447 | 7648 4448 | 4177 4449 | 7472 4450 | 8572 4451 | 27 4452 | 2339 4453 | 4303 4454 | 9208 4455 | 4489 4456 | 10344 4457 | 2389 4458 | 5948 4459 | 2848 4460 | 6229 4461 | 2033 4462 | 7808 4463 | 9650 4464 | 6659 4465 | 8948 4466 | 2568 4467 | 1618 4468 | 2627 4469 | 807 4470 | 3830 4471 | 2066 4472 | 3628 4473 | 7624 4474 | 4271 4475 | 4124 4476 | 10245 4477 | 2051 4478 | 7141 4479 | 1313 4480 | 4472 4481 | 7540 4482 | 10303 4483 | 10593 4484 | 1473 4485 | 1943 4486 | 5461 4487 | 3307 4488 | 1455 4489 | 7239 4490 | 10053 4491 | 4806 4492 | 8367 4493 | 417 4494 | 6747 4495 | 3237 4496 | 8719 4497 | 2019 4498 | 1397 4499 | 3481 4500 | 4920 4501 | 10209 4502 | 4646 4503 | 4740 4504 | 3514 4505 | 7315 4506 | 580 4507 | 10302 4508 | 7316 4509 | 2835 4510 | 7433 4511 | 7512 4512 | 7436 4513 | 676 4514 | 9729 4515 | 3775 4516 | 5500 4517 | 5370 4518 | 7941 4519 | 5213 4520 | 7969 4521 | 6730 4522 | 10340 4523 | 6763 4524 | 8652 4525 | 4730 4526 | 5843 4527 | 5239 4528 | 3546 4529 | 9000 4530 | 3266 4531 | 6450 4532 | 9549 4533 | 7361 4534 | 1390 4535 | 7885 4536 | 8050 4537 | 1302 4538 | 9725 4539 | 6178 4540 | 4192 4541 | 9843 4542 | 8104 4543 | 870 4544 | 10243 4545 | 5288 4546 | 6054 4547 | 7344 4548 | 10150 4549 | 957 4550 | 4843 4551 | 2061 4552 | 7742 4553 | 8557 4554 | 2882 4555 | 8742 4556 | 6640 4557 | 678 4558 | 9660 4559 | 2211 4560 | 75 4561 | 6500 4562 | 6222 4563 | 4467 4564 | 445 4565 | 6750 4566 | 10314 4567 | 2052 4568 | 6582 4569 | 6909 4570 | 6809 4571 | 5678 4572 | 2828 4573 | 2230 4574 | 7212 4575 | 5090 4576 | 8416 4577 | 1223 4578 | 2104 4579 | 1702 4580 | 1547 4581 | 2727 4582 | 2863 4583 | 7971 4584 | 1451 4585 | 7838 4586 | 1475 4587 | 3033 4588 | 2964 4589 | 3418 4590 | 6012 4591 | 7031 4592 | 2919 4593 | 9304 4594 | 10170 4595 | 3412 4596 | 3565 4597 | 5411 4598 | 1682 4599 | 1371 4600 | 7398 4601 | 3045 4602 | 8312 4603 | 4055 4604 | 2032 4605 | 4870 4606 | 1277 4607 | 7577 4608 | 10366 4609 | 8863 4610 | 9091 4611 | 8036 4612 | 9354 4613 | 2381 4614 | 8662 4615 | 4095 4616 | 7712 4617 | 7095 4618 | 2521 4619 | 6934 4620 | 9268 4621 | 5621 4622 | 10543 4623 | 2267 4624 | 1468 4625 | 2128 4626 | 2928 4627 | 8904 4628 | 9359 4629 | 4954 4630 | 2787 4631 | 5941 4632 | 1266 4633 | 10182 4634 | 8981 4635 | 1742 4636 | 7706 4637 | 1636 4638 | 9411 4639 | 8805 4640 | 269 4641 | 10564 4642 | 10018 4643 | 5540 4644 | 490 4645 | 7251 4646 | 3264 4647 | 7104 4648 | 1875 4649 | 3465 4650 | 5961 4651 | 2071 4652 | 5925 4653 | 6100 4654 | 5368 4655 | 7894 4656 | 1653 4657 | 10113 4658 | 7313 4659 | 4092 4660 | 3066 4661 | 1274 4662 | 8566 4663 | 699 4664 | 9919 4665 | 8923 4666 | 3727 4667 | 8627 4668 | 5589 4669 | 5067 4670 | 7942 4671 | 10317 4672 | 9758 4673 | 4952 4674 | 954 4675 | 1191 4676 | 9986 4677 | 4555 4678 | 7792 4679 | 4808 4680 | 245 4681 | 5087 4682 | 4155 4683 | 7499 4684 | 7719 4685 | 1066 4686 | 2390 4687 | 3406 4688 | 2894 4689 | 3093 4690 | 2217 4691 | 9089 4692 | 9148 4693 | 971 4694 | 67 4695 | 5251 4696 | 7101 4697 | 4510 4698 | 2153 4699 | 2831 4700 | 4948 4701 | 9962 4702 | 6259 4703 | 1580 4704 | 1170 4705 | 6027 4706 | 4252 4707 | 6667 4708 | 1015 4709 | 2890 4710 | 1601 4711 | 6058 4712 | 1393 4713 | 4357 4714 | 1780 4715 | 9550 4716 | 6129 4717 | 6112 4718 | 9571 4719 | 7988 4720 | 9692 4721 | 2417 4722 | 10199 4723 | 218 4724 | 1794 4725 | 4072 4726 | 4769 4727 | 5064 4728 | 309 4729 | 9958 4730 | 13 4731 | 10538 4732 | 8135 4733 | 5084 4734 | 1936 4735 | 1334 4736 | 4442 4737 | 1608 4738 | 488 4739 | 3928 4740 | 6115 4741 | 9842 4742 | 10008 4743 | 8758 4744 | 2945 4745 | 6314 4746 | 7732 4747 | 1532 4748 | 9195 4749 | 6004 4750 | 9421 4751 | 2973 4752 | 9858 4753 | 5610 4754 | 7386 4755 | 6560 4756 | 5182 4757 | 9653 4758 | 3125 4759 | 2883 4760 | 3373 4761 | 10323 4762 | 2781 4763 | 4636 4764 | 2843 4765 | 440 4766 | 10281 4767 | 5338 4768 | 4288 4769 | 3943 4770 | 6233 4771 | 6752 4772 | 7336 4773 | 3402 4774 | 10485 4775 | 3120 4776 | 2194 4777 | 10517 4778 | 4266 4779 | 144 4780 | 5744 4781 | 5603 4782 | 1479 4783 | 7421 4784 | 5262 4785 | 4491 4786 | 5877 4787 | 2510 4788 | 8376 4789 | 9050 4790 | 6559 4791 | 5605 4792 | 5079 4793 | 3571 4794 | 780 4795 | 8499 4796 | 6431 4797 | 1014 4798 | 3609 4799 | 6181 4800 | 8984 4801 | 4610 4802 | 1717 4803 | 5394 4804 | 1361 4805 | 8329 4806 | 6067 4807 | 400 4808 | 8080 4809 | 5907 4810 | 9814 4811 | 6974 4812 | 8787 4813 | 3298 4814 | 8940 4815 | 466 4816 | 100 4817 | 2406 4818 | 10266 4819 | 6016 4820 | 8813 4821 | 7232 4822 | 3068 4823 | 9823 4824 | 6164 4825 | 10455 4826 | 9771 4827 | 526 4828 | 660 4829 | 1499 4830 | 5030 4831 | 199 4832 | 9491 4833 | 826 4834 | 7948 4835 | 9379 4836 | 10528 4837 | 5542 4838 | 553 4839 | 7769 4840 | 6634 4841 | 305 4842 | 7374 4843 | 7643 4844 | 7764 4845 | 4482 4846 | 7858 4847 | 8352 4848 | 8342 4849 | 463 4850 | 3219 4851 | 5789 4852 | 7226 4853 | 8254 4854 | 4225 4855 | 9836 4856 | 5214 4857 | 7455 4858 | 2145 4859 | 6872 4860 | 4622 4861 | 8363 4862 | 140 4863 | 7703 4864 | 8068 4865 | 2559 4866 | 4112 4867 | 9832 4868 | 10311 4869 | 7735 4870 | 10342 4871 | 3600 4872 | 2869 4873 | 9719 4874 | 416 4875 | 4780 4876 | 9767 4877 | 1169 4878 | 3275 4879 | 270 4880 | 2936 4881 | 10414 4882 | 6170 4883 | 2014 4884 | 4220 4885 | 2560 4886 | 1651 4887 | 8493 4888 | 8305 4889 | 3729 4890 | 3680 4891 | 1968 4892 | 6121 4893 | 3637 4894 | 242 4895 | 7387 4896 | 10240 4897 | 8506 4898 | 2254 4899 | 5636 4900 | 917 4901 | 8423 4902 | 8334 4903 | 8463 4904 | 1531 4905 | 4440 4906 | 5810 4907 | 8293 4908 | 7055 4909 | 1324 4910 | 1837 4911 | 8963 4912 | 9153 4913 | 7586 4914 | 8673 4915 | 8901 4916 | 8441 4917 | 4966 4918 | 2115 4919 | 10526 4920 | 645 4921 | 8639 4922 | 313 4923 | 1637 4924 | 3847 4925 | 5691 4926 | 1216 4927 | 6479 4928 | 5400 4929 | 2644 4930 | 2799 4931 | 4028 4932 | 6407 4933 | 10130 4934 | 7802 4935 | 1489 4936 | 9992 4937 | 1460 4938 | 9622 4939 | 5537 4940 | 7334 4941 | 5189 4942 | 9698 4943 | 7326 4944 | 3580 4945 | 9371 4946 | 3719 4947 | 3348 4948 | 10261 4949 | 8905 4950 | 7809 4951 | 446 4952 | 4427 4953 | 4344 4954 | 9863 4955 | 28 4956 | 7555 4957 | 7304 4958 | 907 4959 | 1095 4960 | 7465 4961 | 8957 4962 | 9511 4963 | 2366 4964 | 6081 4965 | 1197 4966 | 2575 4967 | 10337 4968 | 8018 4969 | 4874 4970 | 4605 4971 | 6670 4972 | 10097 4973 | 5223 4974 | 7601 4975 | 9141 4976 | 2803 4977 | 1881 4978 | 434 4979 | 2475 4980 | 4790 4981 | 2931 4982 | 6213 4983 | 6915 4984 | 8013 4985 | 7245 4986 | 3552 4987 | 9909 4988 | 3811 4989 | 7263 4990 | 1768 4991 | 366 4992 | 1991 4993 | 7352 4994 | 1527 4995 | 9702 4996 | 1438 4997 | 3641 4998 | 9063 4999 | 9961 5000 | 3924 5001 | 204 5002 | 2312 5003 | 1792 5004 | 3667 5005 | 7261 5006 | 1010 5007 | 10399 5008 | 4114 5009 | 5814 5010 | 452 5011 | 476 5012 | 10116 5013 | 1317 5014 | 10486 5015 | 8020 5016 | 6469 5017 | 505 5018 | 1282 5019 | 1306 5020 | 5909 5021 | 4761 5022 | 1311 5023 | 5403 5024 | 4046 5025 | 2612 5026 | 4984 5027 | 6832 5028 | 6396 5029 | 3991 5030 | 5742 5031 | 7604 5032 | 3863 5033 | 1874 5034 | 2673 5035 | 7010 5036 | 4701 5037 | 5713 5038 | 4401 5039 | 3138 5040 | 1284 5041 | 5885 5042 | 9183 5043 | 9540 5044 | 5763 5045 | 6334 5046 | 10647 5047 | 7542 5048 | 10599 5049 | 7751 5050 | 1312 5051 | 2074 5052 | 1285 5053 | 3622 5054 | 9984 5055 | 1163 5056 | 799 5057 | 2929 5058 | 10212 5059 | 5303 5060 | 291 5061 | 8409 5062 | 3217 5063 | 7974 5064 | 4527 5065 | 10379 5066 | 1480 5067 | 2515 5068 | 1089 5069 | 3940 5070 | 10031 5071 | 2205 5072 | 8433 5073 | 65 5074 | 2111 5075 | 2167 5076 | 6625 5077 | 2801 5078 | 1134 5079 | 9028 5080 | 9801 5081 | 1667 5082 | 1517 5083 | 8547 5084 | 8255 5085 | 9194 5086 | 7474 5087 | 1157 5088 | 7317 5089 | 5661 5090 | 8528 5091 | 3531 5092 | 10203 5093 | 1680 5094 | 10234 5095 | 9126 5096 | 9121 5097 | 8798 5098 | 7500 5099 | 10283 5100 | 2892 5101 | 4488 5102 | 6425 5103 | 906 5104 | 9905 5105 | 4222 5106 | 9741 5107 | 3101 5108 | 5798 5109 | 3688 5110 | 9177 5111 | 8111 5112 | 1237 5113 | 2101 5114 | 7254 5115 | 5045 5116 | 2094 5117 | 5402 5118 | 1206 5119 | 10450 5120 | 8166 5121 | 3005 5122 | 4136 5123 | 6454 5124 | 3875 5125 | 6583 5126 | 9861 5127 | 585 5128 | 1998 5129 | 8707 5130 | 9942 5131 | 545 5132 | 8980 5133 | 2772 5134 | 4352 5135 | 8455 5136 | 6029 5137 | 7195 5138 | 5148 5139 | 5124 5140 | 4295 5141 | 7567 5142 | 7426 5143 | 8937 5144 | 3083 5145 | 9944 5146 | 4073 5147 | 9114 5148 | 2257 5149 | 3995 5150 | 10499 5151 | 285 5152 | 2152 5153 | 239 5154 | 5483 5155 | 7784 5156 | 10255 5157 | 5120 5158 | 3340 5159 | 7081 5160 | 5716 5161 | 841 5162 | 1408 5163 | 8429 5164 | 626 5165 | 1122 5166 | 10093 5167 | 7676 5168 | 8091 5169 | 2242 5170 | 5785 5171 | 3623 5172 | 4612 5173 | 126 5174 | 7901 5175 | 6041 5176 | 1391 5177 | 4369 5178 | 958 5179 | 10393 5180 | 5153 5181 | 3515 5182 | 8739 5183 | 2686 5184 | 1146 5185 | 2989 5186 | 4515 5187 | 10530 5188 | 4602 5189 | 1213 5190 | 9227 5191 | 81 5192 | 8073 5193 | 2207 5194 | 6434 5195 | 8655 5196 | 1661 5197 | 4348 5198 | 4122 5199 | 7762 5200 | 10325 5201 | 791 5202 | 8169 5203 | 7853 5204 | 10553 5205 | 493 5206 | 3462 5207 | 8796 5208 | 10088 5209 | 6037 5210 | 4249 5211 | 9910 5212 | 282 5213 | 6537 5214 | 5835 5215 | 6133 5216 | 2802 5217 | 6964 5218 | 6992 5219 | 1093 5220 | 8941 5221 | 8368 5222 | 7821 5223 | 4330 5224 | 5794 5225 | 1327 5226 | 3869 5227 | 5434 5228 | 9052 5229 | 2096 5230 | 513 5231 | 4767 5232 | 8351 5233 | 122 5234 | 1566 5235 | 3759 5236 | 4413 5237 | 863 5238 | 7463 5239 | 987 5240 | 5021 5241 | 10571 5242 | 1036 5243 | 4937 5244 | 9717 5245 | 7505 5246 | 4014 5247 | 63 5248 | 189 5249 | 7092 5250 | 3426 5251 | 1022 5252 | 2195 5253 | 2286 5254 | 8064 5255 | 2856 5256 | 5423 5257 | 1561 5258 | 2940 5259 | 5132 5260 | 9796 5261 | 6718 5262 | 7001 5263 | 4869 5264 | 8261 5265 | 5876 5266 | 581 5267 | 608 5268 | 10299 5269 | 4283 5270 | 7816 5271 | 2147 5272 | 9104 5273 | 4529 5274 | 5397 5275 | 8465 5276 | 5659 5277 | 1416 5278 | 8755 5279 | 367 5280 | 5839 5281 | 7549 5282 | 2056 5283 | 2715 5284 | 7342 5285 | 2378 5286 | 4093 5287 | 9446 5288 | 1614 5289 | 8062 5290 | 25 5291 | 1236 5292 | 3474 5293 | 4465 5294 | 4724 5295 | 9524 5296 | 401 5297 | 10260 5298 | 6490 5299 | 7285 5300 | 6387 5301 | 1670 5302 | 4168 5303 | 7565 5304 | 720 5305 | 4294 5306 | 1953 5307 | 9118 5308 | 1860 5309 | 3500 5310 | 4164 5311 | 1024 5312 | 5504 5313 | 5086 5314 | 5164 5315 | 6053 5316 | 2079 5317 | 7946 5318 | 6748 5319 | 2106 5320 | 1039 5321 | 5782 5322 | 9214 5323 | 7794 5324 | 8628 5325 | 4611 5326 | 10500 5327 | 6541 5328 | 9521 5329 | 2133 5330 | 1044 5331 | 4986 5332 | 6876 5333 | 4505 5334 | 6247 5335 | 914 5336 | 5709 5337 | 2297 5338 | 9326 5339 | 5049 5340 | 10487 5341 | 5848 5342 | 2643 5343 | 4779 5344 | 2561 5345 | 10381 5346 | 7749 5347 | 3049 5348 | 10407 5349 | 7225 5350 | 696 5351 | 6624 5352 | 8818 5353 | 5490 5354 | 5757 5355 | 7685 5356 | 6088 5357 | 10504 5358 | 5741 5359 | 6902 5360 | 9765 5361 | 259 5362 | 6911 5363 | 8186 5364 | 8665 5365 | 2721 5366 | 10391 5367 | 6767 5368 | 5528 5369 | 9454 5370 | 9828 5371 | 1137 5372 | 5573 5373 | 644 5374 | 8229 5375 | 3342 5376 | 1292 5377 | 6594 5378 | 5226 5379 | 9761 5380 | 8028 5381 | 10618 5382 | 901 5383 | 7694 5384 | 4680 5385 | 9128 5386 | 10122 5387 | 1741 5388 | 2350 5389 | 9974 5390 | 76 5391 | 10466 5392 | 2670 5393 | 6761 5394 | 1083 5395 | 9672 5396 | 9263 5397 | 2760 5398 | 3554 5399 | 1621 5400 | 1374 5401 | 4647 5402 | 9306 5403 | 2452 5404 | 7100 5405 | 7484 5406 | 10269 5407 | 1791 5408 | 2373 5409 | 4553 5410 | 2084 5411 | 7588 5412 | 8950 5413 | 1279 5414 | 308 5415 | 7688 5416 | 4865 5417 | 4844 5418 | 102 5419 | 3541 5420 | 8589 5421 | 5352 5422 | 1132 5423 | 3235 5424 | 3866 5425 | 1491 5426 | 1143 5427 | 9178 5428 | 4958 5429 | 3151 5430 | 3212 5431 | 7840 5432 | 803 5433 | 3186 5434 | 6822 5435 | 1046 5436 | 7528 5437 | 4175 5438 | 1569 5439 | 438 5440 | 4504 5441 | 7380 5442 | 9248 5443 | 5260 5444 | 970 5445 | 2671 5446 | 5558 5447 | 3468 5448 | 5748 5449 | 4837 5450 | 7127 5451 | 8936 5452 | 16 5453 | 4008 5454 | 9142 5455 | 3556 5456 | 1207 5457 | 8498 5458 | 731 5459 | 5674 5460 | 3467 5461 | 2541 5462 | 157 5463 | 5088 5464 | 7273 5465 | 4360 5466 | 5977 5467 | 2551 5468 | 200 5469 | 3601 5470 | 7944 5471 | 6826 5472 | 3697 5473 | 6192 5474 | 6742 5475 | 8130 5476 | 5934 5477 | 4988 5478 | 6453 5479 | 5708 5480 | 10546 5481 | 6000 5482 | 5676 5483 | 8401 5484 | 3651 5485 | 3958 5486 | 7670 5487 | 5774 5488 | 208 5489 | 594 5490 | 927 5491 | 8749 5492 | 666 5493 | 8669 5494 | 5915 5495 | 3407 5496 | 5618 5497 | 7644 5498 | 1254 5499 | 8592 5500 | 10016 5501 | 7490 5502 | 2078 5503 | 7079 5504 | 7634 5505 | 1464 5506 | 873 5507 | 5173 5508 | 9266 5509 | 7660 5510 | 6801 5511 | 2630 5512 | 5169 5513 | 2717 5514 | 7916 5515 | 5553 5516 | 1372 5517 | 2038 5518 | 9878 5519 | 10139 5520 | 5883 5521 | 741 5522 | 1151 5523 | 3361 5524 | 4238 5525 | 9158 5526 | 6203 5527 | 9525 5528 | 5287 5529 | 3895 5530 | 2009 5531 | 5753 5532 | 910 5533 | 6455 5534 | 5983 5535 | 571 5536 | 1255 5537 | 3347 5538 | 981 5539 | 9290 5540 | 3841 5541 | 241 5542 | 2280 5543 | 2579 5544 | 4783 5545 | 3452 5546 | 734 5547 | 764 5548 | 4276 5549 | 9959 5550 | 6087 5551 | 9020 5552 | 9604 5553 | 2348 5554 | 3799 5555 | 7328 5556 | 2123 5557 | 1833 5558 | 8699 5559 | 2324 5560 | 9803 5561 | 925 5562 | 2844 5563 | 8320 5564 | 5863 5565 | 8341 5566 | 502 5567 | 2489 5568 | 2523 5569 | 3150 5570 | 8761 5571 | 3042 5572 | 900 5573 | 4054 5574 | 8680 5575 | 7875 5576 | 6900 5577 | 5026 5578 | 2631 5579 | 4653 5580 | 7124 5581 | 5664 5582 | 4443 5583 | 1369 5584 | 10506 5585 | 6820 5586 | 3833 5587 | 6649 5588 | 2411 5589 | 8873 5590 | 3511 5591 | 9726 5592 | 3927 5593 | 8245 5594 | 3189 5595 | 2637 5596 | 762 5597 | 10138 5598 | 7183 5599 | 3250 5600 | 3013 5601 | 1352 5602 | 213 5603 | 4511 5604 | 2591 5605 | 4469 5606 | 9387 5607 | 1128 5608 | 7684 5609 | 8278 5610 | 8191 5611 | 6175 5612 | 448 5613 | 9349 5614 | 472 5615 | 3025 5616 | 7623 5617 | 10349 5618 | 6603 5619 | 8484 5620 | 8949 5621 | 1828 5622 | 670 5623 | 2364 5624 | 7456 5625 | 2043 5626 | 3287 5627 | 41 5628 | 8494 5629 | 10658 5630 | 7177 5631 | 7343 5632 | 3665 5633 | 10179 5634 | 8427 5635 | 6528 5636 | 5373 5637 | 158 5638 | 10055 5639 | 3698 5640 | 7616 5641 | 4094 5642 | 4888 5643 | 953 5644 | 8001 5645 | 6686 5646 | 1108 5647 | 5344 5648 | 4904 5649 | 1767 5650 | 7610 5651 | 5620 5652 | 2830 5653 | 6555 5654 | 138 5655 | 4585 5656 | 9742 5657 | 681 5658 | 9096 5659 | 3224 5660 | 10540 5661 | 8615 5662 | 3346 5663 | 7414 5664 | 8533 5665 | 3890 5666 | 2316 5667 | 5652 5668 | 3147 5669 | 8090 5670 | 2582 5671 | 5219 5672 | 2455 5673 | 6586 5674 | 6408 5675 | 1848 5676 | 3135 5677 | 1194 5678 | 9607 5679 | 7519 5680 | 5670 5681 | 314 5682 | 1632 5683 | 1161 5684 | 3062 5685 | 5319 5686 | 8196 5687 | 4564 5688 | 8999 5689 | 3629 5690 | 8350 5691 | 2623 5692 | 6893 5693 | 7543 5694 | 2912 5695 | 2638 5696 | 5184 5697 | 7529 5698 | 8693 5699 | 1994 5700 | 2769 5701 | 1559 5702 | 2668 5703 | 10213 5704 | 3926 5705 | 8434 5706 | 319 5707 | 5677 5708 | 9403 5709 | 10115 5710 | 7777 5711 | 10447 5712 | 8432 5713 | 8705 5714 | 7395 5715 | 3277 5716 | 8146 5717 | 7048 5718 | 2150 5719 | 7658 5720 | 10576 5721 | 5355 5722 | 91 5723 | 10126 5724 | 8165 5725 | 6631 5726 | 2234 5727 | 9287 5728 | 2362 5729 | 363 5730 | 5127 5731 | 6507 5732 | 9320 5733 | 3606 5734 | 9564 5735 | 7725 5736 | 606 5737 | 1581 5738 | 3003 5739 | 10405 5740 | 5497 5741 | 7180 5742 | 7699 5743 | 9721 5744 | 798 5745 | 3098 5746 | 3143 5747 | 1174 5748 | 8596 5749 | 887 5750 | 4040 5751 | 4886 5752 | 2421 5753 | 7167 5754 | 3244 5755 | 3457 5756 | 7006 5757 | 10279 5758 | 415 5759 | 2039 5760 | 5703 5761 | 205 5762 | 5926 5763 | 6958 5764 | 2967 5765 | 8497 5766 | 575 5767 | 2485 5768 | 5323 5769 | 6157 5770 | 891 5771 | 9964 5772 | 7906 5773 | 2857 5774 | 4320 5775 | 3539 5776 | 10600 5777 | 4109 5778 | 8173 5779 | 886 5780 | 4877 5781 | 8093 5782 | 9408 5783 | 6340 5784 | 2953 5785 | 1097 5786 | 5777 5787 | 5014 5788 | 10355 5789 | 7730 5790 | 9163 5791 | 2310 5792 | 3954 5793 | 2547 5794 | 6978 5795 | 1071 5796 | 8584 5797 | 876 5798 | 10524 5799 | 2062 5800 | 6897 5801 | 10410 5802 | 8683 5803 | 9316 5804 | 5221 5805 | 3380 5806 | 5732 5807 | 8081 5808 | 9895 5809 | 7460 5810 | 653 5811 | 1732 5812 | 9180 5813 | 4144 5814 | 839 5815 | 5666 5816 | 10590 5817 | 5543 5818 | 5186 5819 | 7788 5820 | 3449 5821 | 7715 5822 | 4816 5823 | 3893 5824 | 10154 5825 | 10315 5826 | 3216 5827 | 7950 5828 | 9763 5829 | 10108 5830 | 7144 5831 | 10373 5832 | 6339 5833 | 2581 5834 | 4831 5835 | 949 5836 | 3057 5837 | 8038 5838 | 10052 5839 | 9645 5840 | 5838 5841 | 9452 5842 | 2524 5843 | 4880 5844 | 5369 5845 | 2498 5846 | 5609 5847 | 3517 5848 | 979 5849 | 6914 5850 | 451 5851 | 2583 5852 | 1939 5853 | 7945 5854 | 2881 5855 | 4257 5856 | 7470 5857 | 10389 5858 | 5256 5859 | 7582 5860 | 10645 5861 | 3054 5862 | 2120 5863 | 9220 5864 | 570 5865 | 9029 5866 | 1205 5867 | 3032 5868 | 3988 5869 | 6616 5870 | 6552 5871 | 955 5872 | 2384 5873 | 6515 5874 | 6922 5875 | 10094 5876 | 7630 5877 | 9085 5878 | 7876 5879 | 2504 5880 | 10095 5881 | 1064 5882 | 4720 5883 | 6841 5884 | 273 5885 | 7756 5886 | 2409 5887 | 7965 5888 | 7598 5889 | 10635 5890 | 4404 5891 | 6707 5892 | 4628 5893 | 1843 5894 | 6662 5895 | 1959 5896 | 8078 5897 | 8808 5898 | 9222 5899 | 9203 5900 | 5061 5901 | 3950 5902 | 1849 5903 | 1529 5904 | 10207 5905 | 6009 5906 | 3618 5907 | 7995 5908 | 7062 5909 | 4823 5910 | 5436 5911 | 5807 5912 | 3386 5913 | 3754 5914 | 8303 5915 | 1957 5916 | 3848 5917 | 3979 5918 | 3776 5919 | 3375 5920 | 2012 5921 | 6954 5922 | 5599 5923 | 2992 5924 | 1283 5925 | 5688 5926 | 8791 5927 | 8492 5928 | 2479 5929 | 1958 5930 | 7914 5931 | 5057 5932 | 4977 5933 | 7243 5934 | 10328 5935 | 1578 5936 | 2971 5937 | 4235 5938 | 2821 5939 | 10049 5940 | 2309 5941 | 754 5942 | 3717 5943 | 999 5944 | 1810 5945 | 3885 5946 | 1605 5947 | 10444 5948 | 6446 5949 | 7173 5950 | 709 5951 | 7439 5952 | 9587 5953 | 529 5954 | 8682 5955 | 9552 5956 | 330 5957 | 10465 5958 | 1079 5959 | 7646 5960 | 7741 5961 | 1094 5962 | 2742 5963 | 2684 5964 | 5836 5965 | 6998 5966 | 7337 5967 | 4298 5968 | 7560 5969 | 1840 5970 | 3813 5971 | 1281 5972 | 2780 5973 | 4384 5974 | 4987 5975 | 9640 5976 | 6437 5977 | 1521 5978 | 7687 5979 | 8532 5980 | 6077 5981 | 3908 5982 | 9478 5983 | 4059 5984 | 6266 5985 | 9835 5986 | 1119 5987 | 442 5988 | 7878 5989 | 6806 5990 | 6182 5991 | 8461 5992 | 5760 5993 | 5828 5994 | 659 5995 | 9087 5996 | 913 5997 | 4414 5998 | 8426 5999 | 3952 6000 | 7804 6001 | 4400 6002 | 10215 6003 | 4392 6004 | 6812 6005 | 119 6006 | 3022 6007 | 6212 6008 | 664 6009 | 9637 6010 | 5930 6011 | 8487 6012 | 4218 6013 | 6080 6014 | 625 6015 | 10475 6016 | 6561 6017 | 5427 6018 | 10210 6019 | 9040 6020 | 334 6021 | 4065 6022 | 7448 6023 | 2118 6024 | 2791 6025 | 2114 6026 | 1789 6027 | 2200 6028 | 33 6029 | 5939 6030 | 3358 6031 | 9994 6032 | 9129 6033 | 9991 6034 | 9517 6035 | 9854 6036 | 3575 6037 | 360 6038 | 9046 6039 | 9293 6040 | 3966 6041 | 6045 6042 | 4380 6043 | 6197 6044 | 4208 6045 | 8005 6046 | 2794 6047 | 2933 6048 | 40 6049 | 851 6050 | 7760 6051 | 3194 6052 | 92 6053 | 8236 6054 | 1858 6055 | 7805 6056 | 1166 6057 | 2199 6058 | 10312 6059 | 2804 6060 | 196 6061 | 6652 6062 | 1878 6063 | 9285 6064 | 9006 6065 | 8554 6066 | 2975 6067 | 4563 6068 | 7904 6069 | 6320 6070 | 3075 6071 | 5299 6072 | 2578 6073 | 1109 6074 | 8841 6075 | 6596 6076 | 2338 6077 | 7899 6078 | 10164 6079 | 7235 6080 | 3902 6081 | 2023 6082 | 10144 6083 | 4911 6084 | 1294 6085 | 695 6086 | 3288 6087 | 2996 6088 | 6657 6089 | 4039 6090 | 9254 6091 | 2317 6092 | 7379 6093 | 7400 6094 | 8952 6095 | 4210 6096 | 4233 6097 | 9 6098 | 3765 6099 | 3992 6100 | 9165 6101 | 10437 6102 | 1587 6103 | 9062 6104 | 6126 6105 | 388 6106 | 587 6107 | 5150 6108 | 10545 6109 | 9193 6110 | 4812 6111 | 372 6112 | 2507 6113 | 5009 6114 | 2095 6115 | 8648 6116 | 47 6117 | 786 6118 | 7228 6119 | 8382 6120 | 6691 6121 | 9683 6122 | 884 6123 | 7461 6124 | 9496 6125 | 1359 6126 | 274 6127 | 3914 6128 | 7677 6129 | 9781 6130 | 5161 6131 | 2728 6132 | 7376 6133 | 9086 6134 | 5594 6135 | 4709 6136 | 9651 6137 | 5971 6138 | 4345 6139 | 1082 6140 | 5729 6141 | 9435 6142 | 10354 6143 | 1028 6144 | 2425 6145 | 9201 6146 | 3056 6147 | 8661 6148 | 7886 6149 | 7492 6150 | 6886 6151 | 6421 6152 | 4652 6153 | 8515 6154 | 3752 6155 | 8253 6156 | 7972 6157 | 7283 6158 | 8843 6159 | 10147 6160 | 9270 6161 | 5003 6162 | 4342 6163 | 4868 6164 | 3018 6165 | 5865 6166 | 6510 6167 | 5631 6168 | 8198 6169 | 172 6170 | 2137 6171 | 6345 6172 | 9352 6173 | 7513 6174 | 4898 6175 | 8344 6176 | 470 6177 | 10491 6178 | 1660 6179 | 238 6180 | 10220 6181 | 1404 6182 | 10300 6183 | 467 6184 | 3176 6185 | 10610 6186 | 4828 6187 | 7126 6188 | 10549 6189 | 4565 6190 | 9215 6191 | 6653 6192 | 7934 6193 | 8299 6194 | 10494 6195 | 8193 6196 | 2949 6197 | 1278 6198 | 7205 6199 | 3796 6200 | 10643 6201 | 2963 6202 | 6516 6203 | 524 6204 | 8899 6205 | 10352 6206 | 4503 6207 | 8853 6208 | 5037 6209 | 3755 6210 | 4102 6211 | 8726 6212 | 4688 6213 | 5162 6214 | 5598 6215 | 6903 6216 | 9139 6217 | 6531 6218 | 5965 6219 | 4002 6220 | 3513 6221 | 4591 6222 | 5850 6223 | 8355 6224 | 7409 6225 | 280 6226 | 1406 6227 | 5154 6228 | 8553 6229 | 8300 6230 | 1710 6231 | 6423 6232 | 2625 6233 | 7299 6234 | 4158 6235 | 5447 6236 | 1704 6237 | 1749 6238 | 4814 6239 | 5211 6240 | 9673 6241 | 8446 6242 | 1871 6243 | 5457 6244 | 8454 6245 | 3586 6246 | 6842 6247 | 4346 6248 | 5268 6249 | 4178 6250 | 7267 6251 | 49 6252 | 5043 6253 | 464 6254 | 8398 6255 | 7982 6256 | 8279 6257 | 3901 6258 | 10534 6259 | 1096 6260 | 7292 6261 | 5422 6262 | 3700 6263 | 246 6264 | 8654 6265 | 5616 6266 | 9997 6267 | 9250 6268 | 2007 6269 | 2741 6270 | 10532 6271 | 9455 6272 | 5114 6273 | 8846 6274 | 266 6275 | 1002 6276 | 6568 6277 | 10514 6278 | 5932 6279 | 1970 6280 | 8586 6281 | 3981 6282 | 9690 6283 | 1987 6284 | 6525 6285 | 7830 6286 | 3269 6287 | 4632 6288 | 4748 6289 | 804 6290 | 6241 6291 | 3043 6292 | 2163 6293 | 7320 6294 | 7066 6295 | 2423 6296 | 5764 6297 | 6268 6298 | 2050 6299 | 5023 6300 | 10204 6301 | 1437 6302 | 5535 6303 | 9749 6304 | 8539 6305 | 7581 6306 | 9848 6307 | 3381 6308 | 4367 6309 | 3720 6310 | 7917 6311 | 1190 6312 | 4941 6313 | 8720 6314 | 1433 6315 | 2369 6316 | 10121 6317 | 5984 6318 | 3499 6319 | 5350 6320 | 2357 6321 | 2805 6322 | 9312 6323 | 1220 6324 | 104 6325 | 4895 6326 | 6782 6327 | 5772 6328 | 9186 6329 | 5759 6330 | 516 6331 | 3236 6332 | 178 6333 | 174 6334 | 4833 6335 | 2089 6336 | 9459 6337 | 2410 6338 | 279 6339 | 2775 6340 | 2085 6341 | 8437 6342 | 2260 6343 | 7800 6344 | 4547 6345 | 6293 6346 | 10163 6347 | 10377 6348 | 9109 6349 | 3617 6350 | 7724 6351 | 10124 6352 | 1582 6353 | 3696 6354 | 8928 6355 | 813 6356 | 8837 6357 | 4256 6358 | 6392 6359 | 5998 6360 | 5683 6361 | 4537 6362 | 3091 6363 | 3146 6364 | 1530 6365 | 8552 6366 | 9850 6367 | 6768 6368 | 9904 6369 | 3050 6370 | 610 6371 | 10579 6372 | 9898 6373 | 920 6374 | 7556 6375 | 2750 6376 | 1882 6377 | 861 6378 | 5808 6379 | 2396 6380 | 8706 6381 | 3410 6382 | 2543 6383 | 73 6384 | 3716 6385 | 4996 6386 | 796 6387 | 9869 6388 | 10657 6389 | 6865 6390 | 6513 6391 | 8476 6392 | 7427 6393 | 7613 6394 | 7962 6395 | 3772 6396 | 9314 6397 | 3855 6398 | 4899 6399 | 9812 6400 | 1885 6401 | 879 6402 | 4355 6403 | 8822 6404 | 7007 6405 | 4901 6406 | 8105 6407 | 8617 6408 | 5551 6409 | 9185 6410 | 10267 6411 | 4665 6412 | 3352 6413 | 5233 6414 | 7617 6415 | 507 6416 | 10413 6417 | 7574 6418 | 9132 6419 | 1835 6420 | 7678 6421 | 897 6422 | 3643 6423 | 2754 6424 | 5167 6425 | 6709 6426 | 7078 6427 | 5408 6428 | 3285 6429 | 7561 6430 | 6789 6431 | 1786 6432 | 8609 6433 | 5360 6434 | 5649 6435 | 8858 6436 | 5864 6437 | 4153 6438 | 5870 6439 | 2438 6440 | 6811 6441 | 2287 6442 | 3076 6443 | 5181 6444 | 8612 6445 | 560 6446 | 6456 6447 | 8902 6448 | 3763 6449 | 875 6450 | 4500 6451 | 7362 6452 | 919 6453 | 6514 6454 | 4834 6455 | 1604 6456 | 9206 6457 | 10021 6458 | 9077 6459 | 6493 6460 | 5687 6461 | 6791 6462 | 4960 6463 | 3656 6464 | 9694 6465 | 2633 6466 | 7938 6467 | 3967 6468 | 5145 6469 | 7511 6470 | 4017 6471 | 1844 6472 | 9675 6473 | 1411 6474 | 8307 6475 | 3802 6476 | 1512 6477 | 5952 6478 | 6990 6479 | 8859 6480 | 3253 6481 | 8048 6482 | 598 6483 | 9013 6484 | 8960 6485 | 3124 6486 | 693 6487 | 7391 6488 | 1615 6489 | 7806 6490 | 4763 6491 | 10227 6492 | 7370 6493 | 8649 6494 | 5562 6495 | 5911 6496 | 1138 6497 | 1202 6498 | 8224 6499 | 6661 6500 | 1054 6501 | 7793 6502 | 10305 6503 | 10384 6504 | 3787 6505 | 8670 6506 | 7943 6507 | 2065 6508 | 3324 6509 | 10451 6510 | 3819 6511 | 7156 6512 | 1075 6513 | 7584 6514 | 3249 6515 | 55 6516 | 1820 6517 | 4197 6518 | 2186 6519 | 7403 6520 | 7639 6521 | 7052 6522 | 4211 6523 | 7429 6524 | 6146 6525 | 5725 6526 | 7753 6527 | 7303 6528 | 9479 6529 | 3548 6530 | 7907 6531 | 7323 6532 | 4143 6533 | 8119 6534 | 5270 6535 | 6447 6536 | 1796 6537 | 7766 6538 | 10609 6539 | 632 6540 | 6218 6541 | 6592 6542 | 3059 6543 | 2458 6544 | 1513 6545 | 3738 6546 | 9987 6547 | 8697 6548 | 7024 6549 | 1567 6550 | 8890 6551 | 4419 6552 | 1276 6553 | 4825 6554 | 5051 6555 | 9530 6556 | 9769 6557 | 8972 6558 | 6854 6559 | 3842 6560 | 2705 6561 | 3711 6562 | 1500 6563 | 6676 6564 | 4483 6565 | 1012 6566 | 8660 6567 | 6850 6568 | 6644 6569 | 8900 6570 | 10348 6571 | 6677 6572 | 8785 6573 | 8286 6574 | 1486 6575 | 5279 6576 | 2371 6577 | 6419 6578 | 10508 6579 | 4634 6580 | 8810 6581 | 3366 6582 | 5203 6583 | 6892 6584 | 2563 6585 | 5634 6586 | 10141 6587 | 4694 6588 | 7417 6589 | 2646 6590 | 5548 6591 | 3736 6592 | 3274 6593 | 3764 6594 | 4302 6595 | 6574 6596 | 9709 6597 | 6281 6598 | 9284 6599 | 6776 6600 | 2573 6601 | 7236 6602 | 3820 6603 | 6979 6604 | 4139 6605 | 8827 6606 | 6816 6607 | 6399 6608 | 3534 6609 | 6117 6610 | 7635 6611 | 2690 6612 | 288 6613 | 725 6614 | 10205 6615 | 95 6616 | 5950 6617 | 209 6618 | 2001 6619 | 2486 6620 | 673 6621 | 7770 6622 | 5987 6623 | 1382 6624 | 3897 6625 | 3079 6626 | 952 6627 | 5783 6628 | 4787 6629 | 3451 6630 | 1611 6631 | 8069 6632 | 7366 6633 | 9210 6634 | 6852 6635 | 8070 6636 | 4929 6637 | 1187 6638 | 4078 6639 | 4594 6640 | 9847 6641 | 6338 6642 | 3542 6643 | 6532 6644 | 7018 6645 | 6462 6646 | 9374 6647 | 433 6648 | 8264 6649 | 3630 6650 | 5116 6651 | 9168 6652 | 5399 6653 | 1522 6654 | 5640 6655 | 4631 6656 | 9080 6657 | 5118 6658 | 1203 6659 | 8534 6660 | 1118 6661 | 4096 6662 | 7729 6663 | 7341 6664 | 5947 6665 | 5735 6666 | 4289 6667 | 9042 6668 | 4638 6669 | 3094 6670 | 2586 6671 | 3613 6672 | 3507 6673 | 5300 6674 | 2879 6675 | 690 6676 | 10544 6677 | 6201 6678 | 2427 6679 | 2570 6680 | 9635 6681 | 1156 6682 | 8010 6683 | 6154 6684 | 3786 6685 | 147 6686 | 3588 6687 | 1358 6688 | 9605 6689 | 6461 6690 | 2829 6691 | 6125 6692 | 10135 6693 | 3910 6694 | 1442 6695 | 5549 6696 | 6864 6697 | 8772 6698 | 9021 6699 | 3172 6700 | 730 6701 | 3594 6702 | 5723 6703 | 9877 6704 | 4438 6705 | 2701 6706 | 9251 6707 | 458 6708 | 6039 6709 | 4910 6710 | 1730 6711 | 4457 6712 | 5000 6713 | 4876 6714 | 4681 6715 | 3894 6716 | 2737 6717 | 2676 6718 | 6319 6719 | 4000 6720 | 835 6721 | 9699 6722 | 4049 6723 | 3620 6724 | 692 6725 | 6356 6726 | 668 6727 | 8847 6728 | 1990 6729 | 262 6730 | 5326 6731 | 411 6732 | 1597 6733 | 2941 6734 | 2900 6735 | 4240 6736 | 3929 6737 | 4630 6738 | 150 6739 | 8241 6740 | 7434 6741 | 8851 6742 | 5415 6743 | 10562 6744 | 10339 6745 | 2041 6746 | 9463 6747 | 1769 6748 | 5822 6749 | 858 6750 | 10068 6751 | 5831 6752 | 4259 6753 | 10224 6754 | 1459 6755 | 8180 6756 | 7075 6757 | 9505 6758 | 2554 6759 | 9766 6760 | 7301 6761 | 9364 6762 | 1217 6763 | 2322 6764 | 6358 6765 | 3756 6766 | 5969 6767 | 9922 6768 | 2920 6769 | 3798 6770 | 2356 6771 | 4667 6772 | 6042 6773 | 9123 6774 | 5006 6775 | 8110 6776 | 1861 6777 | 10195 6778 | 8040 6779 | 7846 6780 | 8296 6781 | 5347 6782 | 8086 6783 | 38 6784 | 3156 6785 | 8158 6786 | 5602 6787 | 4970 6788 | 8794 6789 | 1045 6790 | 6240 6791 | 5638 6792 | 1101 6793 | 3460 6794 | 2743 6795 | 7070 6796 | 6237 6797 | 5899 6798 | 64 6799 | 6142 6800 | 7038 6801 | 663 6802 | 9778 6803 | 1545 6804 | 6924 6805 | 414 6806 | 7158 6807 | 8221 6808 | 1062 6809 | 718 6810 | 1267 6811 | 6355 6812 | 1652 6813 | 153 6814 | 6429 6815 | 2472 6816 | 5284 6817 | 569 6818 | 7036 6819 | 10423 6820 | 8834 6821 | 2058 6822 | 9204 6823 | 6138 6824 | 9199 6825 | 8039 6826 | 8046 6827 | 1764 6828 | 9534 6829 | 3559 6830 | 5922 6831 | 3953 6832 | 4011 6833 | 1296 6834 | 9171 6835 | 8502 6836 | 4292 6837 | 647 6838 | 7416 6839 | 3574 6840 | 2487 6841 | 9328 6842 | 2533 6843 | 5617 6844 | 4474 6845 | 3903 6846 | 504 6847 | 143 6848 | 862 6849 | 7553 6850 | 1590 6851 | 2697 6852 | 7333 6853 | 7013 6854 | 5945 6855 | 9159 6856 | 8388 6857 | 10478 6858 | 8449 6859 | 9932 6860 | 4501 6861 | 2696 6862 | 4815 6863 | 135 6864 | 9917 6865 | 7084 6866 | 7217 6867 | 2053 6868 | 4398 6869 | 3215 6870 | 3632 6871 | 4327 6872 | 938 6873 | 6234 6874 | 6725 6875 | 224 6876 | 10019 6877 | 8675 6878 | 1351 6879 | 9562 6880 | 3695 6881 | 1173 6882 | 311 6883 | 6025 6884 | 1256 6885 | 9365 6886 | 1924 6887 | 3345 6888 | 2251 6889 | 6036 6890 | 2143 6891 | 3007 6892 | 1817 6893 | 10551 6894 | 1188 6895 | 7752 6896 | 3999 6897 | 8889 6898 | 377 6899 | 5396 6900 | 8026 6901 | 3289 6902 | 10536 6903 | 2181 6904 | 7029 6905 | 9906 6906 | 931 6907 | 3884 6908 | 1937 6909 | 8237 6910 | 5639 6911 | 4662 6912 | 7269 6913 | 212 6914 | 8678 6915 | 8888 6916 | 3504 6917 | 6853 6918 | 4853 6919 | 9687 6920 | 522 6921 | 1502 6922 | 1144 6923 | 6486 6924 | 5823 6925 | 8865 6926 | 2691 6927 | 9434 6928 | 1951 6929 | 2262 6930 | 10464 6931 | 4341 6932 | 8019 6933 | 10133 6934 | 1633 6935 | 4623 6936 | 1675 6937 | 9573 6938 | 9236 6939 | 6443 6940 | 4409 6941 | 429 6942 | 8725 6943 | 1273 6944 | 6370 6945 | 840 6946 | 7539 6947 | 9689 6948 | 5420 6949 | 827 6950 | 2416 6951 | 4577 6952 | 5938 6953 | 3677 6954 | 8315 6955 | 5958 6956 | 3440 6957 | 9592 6958 | 10427 6959 | 5353 6960 | 6483 6961 | 8740 6962 | 6204 6963 | 8392 6964 | 1465 6965 | 295 6966 | 7682 6967 | 9324 6968 | 6304 6969 | 10298 6970 | 2146 6971 | 6021 6972 | 1310 6973 | 5077 6974 | 7903 6975 | 6275 6976 | 3165 6977 | 6373 6978 | 9071 6979 | 9065 6980 | 2580 6981 | 654 6982 | 2905 6983 | 1466 6984 | 7134 6985 | 6821 6986 | 2529 6987 | 8580 6988 | 9501 6989 | 1716 6990 | 3669 6991 | 3883 6992 | 8962 6993 | 1964 6994 | 3904 6995 | 9303 6996 | 8511 6997 | 2762 6998 | 630 6999 | 7611 7000 | 7600 7001 | 5475 7002 | 9969 7003 | 333 7004 | 5407 7005 | 2859 7006 | 1996 7007 | 5663 7008 | 8411 7009 | 77 7010 | 5860 7011 | 4118 7012 | 2160 7013 | 3710 7014 | 2107 7015 | 1081 7016 | 9205 7017 | 8007 7018 | 8910 7019 | 4794 7020 | 2702 7021 | 3947 7022 | 2327 7023 | 6001 7024 | 3512 7025 | 6017 7026 | 8428 7027 | 4867 7028 | 10386 7029 | 4205 7030 | 3821 7031 | 5027 7032 | 2904 7033 | 5432 7034 | 10242 7035 | 8212 7036 | 517 7037 | 622 7038 | 9465 7039 | 5248 7040 | 8687 7041 | 6195 7042 | 8538 7043 | 708 7044 | 1575 7045 | 7633 7046 | 6535 7047 | 8430 7048 | 10004 7049 | 1141 7050 | 8793 7051 | 6230 7052 | 7661 7053 | 9133 7054 | 9696 7055 | 2614 7056 | 6433 7057 | 5752 7058 | 5643 7059 | 4426 7060 | 2966 7061 | 6435 7062 | 3012 7063 | 6749 7064 | 2122 7065 | 2326 7066 | 3292 7067 | 4530 7068 | 3712 7069 | 8489 7070 | 2494 7071 | 10481 7072 | 6818 7073 | 2910 7074 | 6120 7075 | 455 7076 | 9576 7077 | 3845 7078 | 1519 7079 | 3733 7080 | 7485 7081 | 1153 7082 | 8304 7083 | 5506 7084 | 7 7085 | 386 7086 | 3209 7087 | 8141 7088 | 9590 7089 | 10574 7090 | 5468 7091 | 1257 7092 | 3331 7093 | 6062 7094 | 3888 7095 | 8295 7096 | 2492 7097 | 2785 7098 | 508 7099 | 1042 7100 | 2168 7101 | 1100 7102 | 6326 7103 | 4339 7104 | 5629 7105 | 1801 7106 | 6605 7107 | 1986 7108 | 4608 7109 | 8085 7110 | 3489 7111 | 8663 7112 | 7954 7113 | 2083 7114 | 1407 7115 | 7318 7116 | 8204 7117 | 2029 7118 | 5555 7119 | 3110 7120 | 6346 7121 | 6762 7122 | 8918 7123 | 2466 7124 | 7722 7125 | 427 7126 | 4528 7127 | 9526 7128 | 561 7129 | 7663 7130 | 5345 7131 | 966 7132 | 3732 7133 | 118 7134 | 8885 7135 | 7727 7136 | 1903 7137 | 2825 7138 | 6672 7139 | 4805 7140 | 275 7141 | 6312 7142 | 8838 7143 | 5013 7144 | 6498 7145 | 7462 7146 | 7748 7147 | 398 7148 | 7520 7149 | 9445 7150 | 7629 7151 | 8914 7152 | 9840 7153 | 6451 7154 | 6497 7155 | 6258 7156 | 1063 7157 | 1145 7158 | 3308 7159 | 6861 7160 | 1557 7161 | 9405 7162 | 4328 7163 | 3087 7164 | 7293 7165 | 4521 7166 | 3503 7167 | 5547 7168 | 578 7169 | 2166 7170 | 9134 7171 | 1634 7172 | 912 7173 | 5179 7174 | 8054 7175 | 3937 7176 | 9112 7177 | 5199 7178 | 8415 7179 | 389 7180 | 8577 7181 | 7650 7182 | 10649 7183 | 207 7184 | 1669 7185 | 7262 7186 | 4244 7187 | 4930 7188 | 9657 7189 | 5974 7190 | 988 7191 | 6898 7192 | 3484 7193 | 7973 7194 | 6179 7195 | 5704 7196 | 2874 7197 | 10476 7198 | 1235 7199 | 5518 7200 | 8848 7201 | 7339 7202 | 7710 7203 | 8402 7204 | 7290 7205 | 4009 7206 | 8882 7207 | 5469 7208 | 6844 7209 | 1729 7210 | 124 7211 | 9061 7212 | 2508 7213 | 9711 7214 | 3448 7215 | 8664 7216 | 2442 7217 | 9700 7218 | 6577 7219 | 1889 7220 | 701 7221 | 2903 7222 | 1627 7223 | 10656 7224 | 1386 7225 | 2720 7226 | 2636 7227 | 4361 7228 | 7008 7229 | 6449 7230 | 4082 7231 | 7822 7232 | 960 7233 | 2854 7234 | 10333 7235 | 6043 7236 | 2899 7237 | 4907 7238 | 3769 7239 | 8610 7240 | 738 7241 | 7606 7242 | 1813 7243 | 5336 7244 | 8374 7245 | 3949 7246 | 2716 7247 | 4359 7248 | 2512 7249 | 9226 7250 | 4204 7251 | 3229 7252 | 706 7253 | 7072 7254 | 1583 7255 | 7883 7256 | 10040 7257 | 7491 7258 | 1644 7259 | 10061 7260 | 3415 7261 | 8987 7262 | 1952 7263 | 5679 7264 | 6467 7265 | 6220 7266 | 6076 7267 | 7860 7268 | 5820 7269 | 8607 7270 | 5825 7271 | 1956 7272 | 1330 7273 | 4476 7274 | 10023 7275 | 10520 7276 | 264 7277 | 9593 7278 | 2265 7279 | 7265 7280 | 1250 7281 | 10341 7282 | 193 7283 | 8581 7284 | 4137 7285 | 9551 7286 | 5247 7287 | 9426 7288 | 9829 7289 | 7091 7290 | 9414 7291 | 1870 7292 | 3470 7293 | 10160 7294 | 297 7295 | 6716 7296 | 6743 7297 | 4516 7298 | 9147 7299 | 1655 7300 | 5796 7301 | 8820 7302 | 8262 7303 | 3878 7304 | 8965 7305 | 10180 7306 | 4338 7307 | 7909 7308 | 2258 7309 | 7295 7310 | 1467 7311 | 8043 7312 | 7571 7313 | 1815 7314 | 2774 7315 | 4835 7316 | 7428 7317 | 8399 7318 | 9239 7319 | 5426 7320 | 10585 7321 | 4477 7322 | 8870 7323 | 1483 7324 | 6866 7325 | 7952 7326 | 1713 7327 | 9417 7328 | 3401 7329 | 6140 7330 | 1747 7331 | 535 7332 | 6448 7333 | 1594 7334 | 7182 7335 | 8136 7336 | 1396 7337 | 5277 7338 | 4683 7339 | 9300 7340 | 3459 7341 | 1073 7342 | 9111 7343 | 6819 7344 | 9837 7345 | 3679 7346 | 752 7347 | 146 7348 | 9103 7349 | 2613 7350 | 78 7351 | 6404 7352 | 1171 7353 | 5635 7354 | 10077 7355 | 4854 7356 | 9806 7357 | 2456 7358 | 4879 7359 | 9447 7360 | 7206 7361 | 7382 7362 | 3771 7363 | 10074 7364 | 10603 7365 | 4990 7366 | 3437 7367 | 10461 7368 | 2375 7369 | 6158 7370 | 6511 7371 | 5178 7372 | 8978 7373 | 6283 7374 | 4243 7375 | 44 7376 | 2808 7377 | 9145 7378 | 7548 7379 | 1362 7380 | 1001 7381 | 871 7382 | 6148 7383 | 1259 7384 | 1554 7385 | 1496 7386 | 2273 7387 | 8435 7388 | 7020 7389 | 5273 7390 | 4846 7391 | 1965 7392 | 2201 7393 | 10501 7394 | 4347 7395 | 7992 7396 | 1863 7397 | 10587 7398 | 9437 7399 | 10001 7400 | 7188 7401 | 2124 7402 | 7928 7403 | 10492 7404 | 6161 7405 | 6216 7406 | 10186 7407 | 2624 7408 | 7964 7409 | 454 7410 | 9160 7411 | 62 7412 | 6278 7413 | 6139 7414 | 3443 7415 | 5228 7416 | 5191 7417 | 10229 7418 | 1723 7419 | 7252 7420 | 1602 7421 | 8527 7422 | 1286 7423 | 4368 7424 | 7659 7425 | 3477 7426 | 7638 7427 | 5627 7428 | 3171 7429 | 8340 7430 | 1643 7431 | 2354 7432 | 293 7433 | 9819 7434 | 6010 7435 | 7098 7436 | 2634 7437 | 9793 7438 | 1550 7439 | 9954 7440 | 6693 7441 | 2460 7442 | 6400 7443 | 4316 7444 | 9652 7445 | 5245 7446 | 5750 7447 | 5787 7448 | 7335 7449 | 4745 7450 | 2231 7451 | 3126 7452 | 201 7453 | 5389 7454 | 789 7455 | 7423 7456 | 7889 7457 | 10238 7458 | 4034 7459 | 777 7460 | 613 7461 | 5180 7462 | 733 7463 | 620 7464 | 7279 7465 | 8195 7466 | 2813 7467 | 6307 7468 | 5593 7469 | 422 7470 | 3486 7471 | 5011 7472 | 7480 7473 | 638 7474 | 1676 7475 | 9799 7476 | 4119 7477 | 1038 7478 | 6668 7479 | 3214 7480 | 8588 7481 | 3547 7482 | 1052 7483 | 8769 7484 | 8574 7485 | 8011 7486 | 7364 7487 | 2957 7488 | 302 7489 | 4460 7490 | 4074 7491 | 8407 7492 | 4420 7493 | 5172 7494 | 4388 7495 | 6477 7496 | 8280 7497 | 9390 7498 | 7240 7499 | 8142 7500 | 6860 7501 | 7489 7502 | 1695 7503 | 4185 7504 | 1762 7505 | 7109 7506 | 506 7507 | 2871 7508 | 4881 7509 | 4391 7510 | 4390 7511 | 8008 7512 | 1890 7513 | 8732 7514 | 9470 7515 | 2734 7516 | 4396 7517 | 3693 7518 | 5207 7519 | 772 7520 | 6418 7521 | 1058 7522 | 4578 7523 | 9580 7524 | 7250 7525 | 7921 7526 | 6374 7527 | 1516 7528 | 5856 7529 | 1074 7530 | 4449 7531 | 1380 7532 | 3568 7533 | 4228 7534 | 9244 7535 | 8025 7536 | 2469 7537 | 2282 7538 | 5314 7539 | 3751 7540 | 9528 7541 | 2505 7542 | 2665 7543 | 8809 7544 | 844 7545 | 4532 7546 | 998 7547 | 2834 7548 | 173 7549 | 3199 7550 | 8445 7551 | 1913 7552 | 5671 7553 | 52 7554 | 5693 7555 | 9097 7556 | 7627 7557 | 3351 7558 | 10278 7559 | 869 7560 | 2294 7561 | 6932 7562 | 8458 7563 | 2658 7564 | 5328 7565 | 6758 7566 | 10424 7567 | 5001 7568 | 9008 7569 | 558 7570 | 2382 7571 | 233 7572 | 5916 7573 | 4669 7574 | 6575 7575 | 9458 7576 | 9792 7577 | 7319 7578 | 7441 7579 | 10022 7580 | 3642 7581 | 365 7582 | 9844 7583 | 5477 7584 | 4475 7585 | 651 7586 | 935 7587 | 8263 7588 | 10098 7589 | 254 7590 | 4203 7591 | 353 7592 | 10477 7593 | 1654 7594 | 8132 7595 | 6368 7596 | 7022 7597 | 449 7598 | 3365 7599 | 6611 7600 | 5792 7601 | 5936 7602 | 7202 7603 | 9370 7604 | 5072 7605 | 4184 7606 | 4231 7607 | 9059 7608 | 9480 7609 | 4889 7610 | 3371 7611 | 8849 7612 | 8933 7613 | 9510 7614 | 3801 7615 | 525 7616 | 8488 7617 | 8396 7618 | 3964 7619 | 6265 7620 | 1854 7621 | 2841 7622 | 9737 7623 | 1115 7624 | 6817 7625 | 4349 7626 | 10361 7627 | 9120 7628 | 6075 7629 | 893 7630 | 3273 7631 | 2571 7632 | 10083 7633 | 9260 7634 | 2318 7635 | 9893 7636 | 2753 7637 | 10280 7638 | 10369 7639 | 4221 7640 | 3508 7641 | 473 7642 | 7596 7643 | 8331 7644 | 755 7645 | 2593 7646 | 922 7647 | 6116 7648 | 1933 7649 | 6756 7650 | 2177 7651 | 7632 7652 | 8782 7653 | 9876 7654 | 3810 7655 | 10175 7656 | 6983 7657 | 3576 7658 | 9366 7659 | 5686 7660 | 4430 7661 | 6105 7662 | 2187 7663 | 9839 7664 | 8252 7665 | 3532 7666 | 1932 7667 | 4179 7668 | 7312 7669 | 1476 7670 | 8968 7671 | 3846 7672 | 5699 7673 | 1886 7674 | 3931 7675 | 5955 7676 | 8831 7677 | 6069 7678 | 301 7679 | 2827 7680 | 483 7681 | 4962 7682 | 2059 7683 | 4885 7684 | 2063 7685 | 4804 7686 | 751 7687 | 6049 7688 | 7170 7689 | 7289 7690 | 4535 7691 | 6091 7692 | 10286 7693 | 10131 7694 | 8887 7695 | 148 7696 | 9512 7697 | 9105 7698 | 7736 7699 | 8738 7700 | 1572 7701 | 10145 7702 | 2392 7703 | 10120 7704 | 3173 7705 | 6796 7706 | 8624 7707 | 10248 7708 | 2606 7709 | 2577 7710 | 8815 7711 | 7723 7712 | 8631 7713 | 9500 7714 | 7418 7715 | 5224 7716 | 843 7717 | 1226 7718 | 231 7719 | 10513 7720 | 9162 7721 | 7864 7722 | 8599 7723 | 4558 7724 | 7993 7725 | 10246 7726 | 4496 7727 | 4965 7728 | 6231 7729 | 66 7730 | 2405 7731 | 6022 7732 | 6171 7733 | 6030 7734 | 9949 7735 | 2025 7736 | 1629 7737 | 8671 7738 | 2735 7739 | 4142 7740 | 3791 7741 | 3444 7742 | 5071 7743 | 5786 7744 | 1830 7745 | 2403 7746 | 903 7747 | 338 7748 | 1494 7749 | 4992 7750 | 5101 7751 | 5975 7752 | 3078 7753 | 256 7754 | 9586 7755 | 8712 7756 | 557 7757 | 7128 7758 | 4362 7759 | 7392 7760 | 6607 7761 | 243 7762 | 4900 7763 | 5035 7764 | 7888 7765 | 10395 7766 | 1381 7767 | 1585 7768 | 7089 7769 | 3619 7770 | 1914 7771 | 7997 7772 | 2632 7773 | 10456 7774 | 6937 7775 | 3283 7776 | 267 7777 | 2755 7778 | 9759 7779 | 10136 7780 | 6566 7781 | 8348 7782 | 9291 7783 | 3422 7784 | 3872 7785 | 6663 7786 | 6103 7787 | 10244 7788 | 5143 7789 | 7915 7790 | 7502 7791 | 8222 7792 | 4435 7793 | 5073 7794 | 1504 7795 | 6772 7796 | 348 7797 | 5025 7798 | 6341 7799 | 8748 7800 | 6389 7801 | 3706 7802 | 5560 7803 | 6465 7804 | 4182 7805 | 2342 7806 | 10228 7807 | 8464 7808 | 8473 7809 | 3149 7810 | 5502 7811 | 1245 7812 | 5217 7813 | 9014 7814 | 162 7815 | 4372 7816 | 80 7817 | 431 7818 | 3668 7819 | 1955 7820 | 1195 7821 | 4663 7822 | 7168 7823 | 4897 7824 | 1503 7825 | 6180 7826 | 477 7827 | 4135 7828 | 2712 7829 | 2188 7830 | 6792 7831 | 1346 7832 | 6579 7833 | 1758 7834 | 3907 7835 | 7160 7836 | 5496 7837 | 4934 7838 | 1596 7839 | 1032 7840 | 8894 7841 | 2046 7842 | 8529 7843 | 9879 7844 | 8964 7845 | 6371 7846 | 3364 7847 | 2189 7848 | 8325 7849 | 9436 7850 | 9620 7851 | 4590 7852 | 10398 7853 | 9674 7854 | 657 7855 | 8874 7856 | 8380 7857 | 6143 7858 | 8637 7859 | 5821 7860 | 565 7861 | 1891 7862 | 4239 7863 | 6708 7864 | 3968 7865 | 4103 7866 | 8983 7867 | 5385 7868 | 6208 7869 | 2864 7870 | 1329 7871 | 6805 7872 | 6741 7873 | 5565 7874 | 5460 7875 | 5419 7876 | 5433 7877 | 39 7878 | 2127 7879 | 8579 7880 | 3286 7881 | 9871 7882 | 9024 7883 | 10533 7884 | 6173 7885 | 7425 7886 | 1751 7887 | 9928 7888 | 9633 7889 | 1803 7890 | 9561 7891 | 8037 7892 | 6519 7893 | 5209 7894 | 8795 7895 | 8883 7896 | 3521 7897 | 9289 7898 | 1678 7899 | 7260 7900 | 9373 7901 | 304 7902 | 5569 7903 | 4512 7904 | 3115 7905 | 4679 7906 | 8824 7907 | 6549 7908 | 5651 7909 | 2250 7910 | 2292 7911 | 5413 7912 | 3660 7913 | 7246 7914 | 9419 7915 | 9915 7916 | 9684 7917 | 1931 7918 | 4775 7919 | 1609 7920 | 7278 7921 | 1872 7922 | 4540 7923 | 346 7924 | 1775 7925 | 4025 7926 | 5237 7927 | 6445 7928 | 7233 7929 | 2726 7930 | 136 7931 | 6544 7932 | 96 7933 | 5614 7934 | 4010 7935 | 7025 7936 | 1026 7937 | 8385 7938 | 1031 7939 | 1048 7940 | 8961 7941 | 10111 7942 | 1184 7943 | 5784 7944 | 9884 7945 | 2954 7946 | 5242 7947 | 2761 7948 | 6209 7949 | 6760 7950 | 2767 7951 | 435 7952 | 10619 7953 | 1821 7954 | 197 7955 | 31 7956 | 7975 7957 | 10471 7958 | 6375 7959 | 3918 7960 | 2044 7961 | 3220 7962 | 156 7963 | 3471 7964 | 4217 7965 | 3074 7966 | 5923 7967 | 1238 7968 | 6664 7969 | 5590 7970 | 9531 7971 | 1630 7972 | 3218 7973 | 10288 7974 | 9257 7975 | 2419 7976 | 10270 7977 | 9010 7978 | 8614 7979 | 3859 7980 | 4939 7981 | 7044 7982 | 3986 7983 | 2470 7984 | 1244 7985 | 5107 7986 | 4042 7987 | 5271 7988 | 8051 7989 | 5372 7990 | 5291 7991 | 6879 7992 | 2414 7993 | 265 7994 | 5805 7995 | 3297 7996 | 10158 7997 | 496 7998 | 4758 7999 | 10110 8000 | 4840 8001 | 5520 8002 | 3621 8003 | 486 8004 | 423 8005 | 624 8006 | 1162 8007 | 5695 8008 | 7919 8009 | 4691 8010 | 1175 8011 | 1834 8012 | 4896 8013 | 806 8014 | 6411 8015 | 3435 8016 | 9723 8017 | 5183 8018 | 7027 8019 | 5721 8020 | 2886 8021 | 1829 8022 | 2161 8023 | 5002 8024 | 7187 8025 | 8453 8026 | 1159 8027 | 8183 8028 | 10042 8029 | 8033 8030 | 1510 8031 | 5968 8032 | 1966 8033 | 4121 8034 | 8895 8035 | 3687 8036 | 627 8037 | 9938 8038 | 9012 8039 | 7097 8040 | 8003 8041 | 3561 8042 | 9299 8043 | 5310 8044 | 2227 8045 | 6827 8046 | 4718 8047 | 4379 8048 | 9582 8049 | 2240 8050 | 6305 8051 | 7071 8052 | 1201 8053 | 5083 8054 | 7609 8055 | 6834 8056 | 10107 8057 | 3699 8058 | 8102 8059 | 10252 8060 | 8801 8061 | 5630 8062 | 9988 8063 | 1748 8064 | 1638 8065 | 2288 8066 | 2103 8067 | 7870 8068 | 8597 8069 | 10208 8070 | 5296 8071 | 8057 8072 | 3955 8073 | 9295 8074 | 634 8075 | 5959 8076 | 2901 8077 | 9009 8078 | 924 8079 | 7209 8080 | 1592 8081 | 1689 8082 | 756 8083 | 5113 8084 | 2969 8085 | 2506 8086 | 8022 8087 | 71 8088 | 9202 8089 | 4716 8090 | 9654 8091 | 7775 8092 | 5873 8093 | 4546 8094 | 8077 8095 | 5944 8096 | 6460 8097 | 2927 8098 | 8906 8099 | 8153 8100 | 340 8101 | 2850 8102 | 10634 8103 | 2656 8104 | 5669 8105 | 3333 8106 | 892 8107 | 2509 8108 | 989 8109 | 6571 8110 | 1734 8111 | 1929 8112 | 8106 8113 | 6061 8114 | 8160 8115 | 5387 8116 | 8406 8117 | 867 8118 | 3788 8119 | 6136 8120 | 3182 8121 | 6149 8122 | 3725 8123 | 5740 8124 | 1365 8125 | 10347 8126 | 5040 8127 | 154 8128 | 7708 8129 | 9149 8130 | 2685 8131 | 9269 8132 | 1339 8133 | 4236 8134 | 9439 8135 | 1508 8136 | 9656 8137 | 10605 8138 | 6870 8139 | 2700 8140 | 3721 8141 | 10613 8142 | 4735 8143 | 7873 8144 | 7196 8145 | 9431 8146 | 9410 8147 | 1708 8148 | 9406 8149 | 3330 8150 | 10313 8151 | 8384 8152 | 2480 8153 | 9475 8154 | 646 8155 | 711 8156 | 994 8157 | 7375 8158 | 5917 8159 | 5624 8160 | 819 8161 | 7506 8162 | 5491 8163 | 2527 8164 | 2990 8165 | 9775 8166 | 2995 8167 | 336 8168 | 3164 8169 | 2136 8170 | 6349 8171 | 564 8172 | 9137 8173 | 1142 8174 | 4818 8175 | 2948 8176 | 1354 8177 | 8201 8178 | 7731 8179 | 3944 8180 | 4149 8181 | 10484 8182 | 3329 8183 | 10651 8184 | 2777 8185 | 2597 8186 | 5559 8187 | 4022 8188 | 7028 8189 | 6357 8190 | 2303 8191 | 500 8192 | 1973 8193 | 4033 8194 | 5285 8195 | 7193 8196 | 2873 8197 | 3387 8198 | 6153 8199 | 1323 8200 | 5953 8201 | 10612 8202 | 7622 8203 | 9849 8204 | 6051 8205 | 9519 8206 | 7349 8207 | 677 8208 | 5024 8209 | 10375 8210 | 9900 8211 | 10561 8212 | 3280 8213 | 2247 8214 | 8602 8215 | 6215 8216 | 9800 8217 | 3686 8218 | 3636 8219 | 7758 8220 | 6858 8221 | 5580 8222 | 69 8223 | 1051 8224 | 10069 8225 | 7937 8226 | 1948 8227 | 7003 8228 | 5655 8229 | 8531 8230 | 7647 8231 | 4821 8232 | 3031 8233 | 3783 8234 | 3335 8235 | 2204 8236 | 8486 8237 | 4981 8238 | 4657 8239 | 5038 8240 | 6779 8241 | 361 8242 | 9591 8243 | 3416 8244 | 5133 8245 | 9679 8246 | 797 8247 | 8647 8248 | 7960 8249 | 5053 8250 | 2179 8251 | 2329 8252 | 2553 8253 | 9356 8254 | 8178 8255 | 247 8256 | 9486 8257 | 6317 8258 | 6074 8259 | 6976 8260 | 1842 8261 | 10425 8262 | 6127 8263 | 2291 8264 | 4258 8265 | 5999 8266 | 2662 8267 | 10282 8268 | 3385 8269 | 10035 8270 | 9318 8271 | 3239 8272 | 837 8273 | 596 8274 | 5267 8275 | 8391 8276 | 7163 8277 | 3951 8278 | 674 8279 | 5331 8280 | 3303 8281 | 6095 8282 | 10563 8283 | 7253 8284 | 5089 8285 | 5376 8286 | 2889 8287 | 7844 8288 | 7291 8289 | 6578 8290 | 4717 8291 | 4864 8292 | 4935 8293 | 54 8294 | 9768 8295 | 6481 8296 | 6751 8297 | 2667 8298 | 6778 8299 | 6162 8300 | 2325 8301 | 6960 8302 | 4756 8303 | 5322 8304 | 10230 8305 | 9614 8306 | 8576 8307 | 2640 8308 | 79 8309 | 222 8310 | 1623 8311 | 216 8312 | 404 8313 | 7626 8314 | 4156 8315 | 6315 8316 | 2042 8317 | 10273 8318 | 2914 8319 | 6795 8320 | 5204 8321 | 1447 8322 | 9535 8323 | 7824 8324 | 4933 8325 | 8773 8326 | 7440 8327 | 5942 8328 | 8766 8329 | 8611 8330 | 10232 8331 | 4306 8332 | 4712 8333 | 1264 8334 | 9516 8335 | 3844 8336 | 6011 8337 | 7837 8338 | 3108 8339 | 9055 8340 | 6773 8341 | 1658 8342 | 4771 8343 | 5215 8344 | 4165 8345 | 26 8346 | 10462 8347 | 3510 8348 | 9176 8349 | 7645 8350 | 1906 8351 | 9816 8352 | 8107 8353 | 10177 8354 | 4100 8355 | 3550 8356 | 822 8357 | 9896 8358 | 9033 8359 | 8731 8360 | 5692 8361 | 2176 8362 | 2248 8363 | 7740 8364 | 7866 8365 | 7181 8366 | 9140 8367 | 9277 8368 | 1534 8369 | 2860 8370 | 5462 8371 | 9860 8372 | 6825 8373 | 8321 8374 | 992 8375 | 6669 8376 | 3781 8377 | 5891 8378 | 2683 8379 | 7042 8380 | 2064 8381 | 3024 8382 | 4554 8383 | 6487 8384 | 3757 8385 | 8359 8386 | 3107 8387 | 1628 8388 | 4123 8389 | 2897 8390 | 4234 8391 | 5937 8392 | 5316 8393 | 4848 8394 | 8684 8395 | 4574 8396 | 9151 8397 | 7121 8398 | 4151 8399 | 5874 8400 | 9094 8401 | 8966 8402 | 6068 8403 | 685 8404 | 1606 8405 | 7082 8406 | 1671 8407 | 1232 8408 | 2224 8409 | 667 8410 | 5524 8411 | 8924 8412 | 7619 8413 | 9985 8414 | 2663 8415 | 9632 8416 | 6405 8417 | 3409 8418 | 2022 8419 | 2918 8420 | 7347 8421 | 1102 8422 | 10442 8423 | 9485 8424 | 3197 8425 | 995 8426 | 1720 8427 | 5218 8428 | 3822 8429 | 4810 8430 | 1691 8431 | 6775 8432 | 7953 8433 | 3092 8434 | 215 8435 | 1041 8436 | 6440 8437 | 6702 8438 | 498 8439 | 740 8440 | 2795 8441 | 1168 8442 | 1972 8443 | 10327 8444 | 2386 8445 | 6943 8446 | 300 8447 | 8713 8448 | 991 8449 | 1501 8450 | 2855 8451 | 3577 8452 | 2923 8453 | 4004 8454 | 1165 8455 | 8030 8456 | 6619 8457 | 3191 8458 | 10046 8459 | 10433 8460 | 9237 8461 | 2878 8462 | 6798 8463 | 1679 8464 | 456 8465 | 2958 8466 | 6132 8467 | 10541 8468 | 8909 8469 | 7583 8470 | 6019 8471 | 3645 8472 | 9678 8473 | 7594 8474 | 10518 8475 | 2283 8476 | 253 8477 | 4786 8478 | 4811 8479 | 946 8480 | 6855 8481 | 1229 8482 | 6494 8483 | 6925 8484 | 1123 8485 | 3276 8486 | 911 8487 | 3131 8488 | 227 8489 | 217 8490 | 2937 8491 | 3722 8492 | 2408 8493 | 9113 8494 | 6269 8495 | 5295 8496 | 5012 8497 | 4994 8498 | 7494 8499 | 1897 8500 | 10338 8501 | 1976 8502 | 4458 8503 | 9030 8504 | 1085 8505 | 9428 8506 | 6361 8507 | 3558 8508 | 8746 8509 | 6534 8510 | 6975 8511 | 10071 8512 | 6473 8513 | 7963 8514 | 5170 8515 | 3294 8516 | 2026 8517 | 7367 8518 | 866 8519 | 1320 8520 | 2183 8521 | 1892 8522 | 1735 8523 | 6488 8524 | 10156 8525 | 2484 8526 | 6327 8527 | 10509 8528 | 9834 8529 | 2206 8530 | 8836 8531 | 2098 8532 | 8327 8533 | 6253 8534 | 1035 8535 | 5799 8536 | 980 8537 | 9933 8538 | 2045 8539 | 6731 8540 | 8347 8541 | 4702 8542 | 7869 8543 | 2695 8544 | 9921 8545 | 829 8546 | 10045 8547 | 3279 8548 | 337 8549 | 2837 8550 | 7855 8551 | 10550 8552 | 8309 8553 | 9727 8554 | 4973 8555 | 3483 8556 | 469 8557 | 382 8558 | 5017 8559 | 743 8560 | 1624 8561 | 2838 8562 | 10617 8563 | 7811 8564 | 7348 8565 | 649 8566 | 3245 8567 | 6810 8568 | 2955 8569 | 3997 8570 | 6759 8571 | 1725 8572 | 8140 8573 | 2865 8574 | 7102 8575 | 568 8576 | 7628 8577 | 745 8578 | 2437 8579 | 4037 8580 | 5056 8581 | 759 8582 | 7309 8583 | 8244 8584 | 4304 8585 | 7815 8586 | 4989 8587 | 8783 8588 | 10142 8589 | 5455 8590 | 9309 8591 | 5163 8592 | 5128 8593 | 2956 8594 | 10368 8595 | 3354 8596 | 9345 8597 | 4108 8598 | 5444 8599 | 7467 8600 | 2797 8601 | 1925 8602 | 10140 8603 | 5811 8604 | 4931 8605 | 325 8606 | 9589 8607 | 6614 8608 | 5445 8609 | 2783 8610 | 6770 8611 | 8947 8612 | 9680 8613 | 335 8614 | 9621 8615 | 5379 8616 | 9068 8617 | 10114 8618 | 9544 8619 | 1289 8620 | 7405 8621 | 7015 8622 | 6296 8623 | 1577 8624 | 6636 8625 | 6144 8626 | 1314 8627 | 8512 8628 | 9975 8629 | 3295 8630 | 3625 8631 | 3933 8632 | 8780 8633 | 2985 8634 | 2174 8635 | 9913 8636 | 6538 8637 | 6214 8638 | 6160 8639 | 7709 8640 | 5960 8641 | 4364 8642 | 1922 8643 | 8548 8644 | 5564 8645 | 6839 8646 | 4394 8647 | 719 8648 | 10638 8649 | 4580 8650 | 1183 8651 | 10159 8652 | 10367 8653 | 8618 8654 | 8059 8655 | 5893 8656 | 4689 8657 | 2474 8658 | 59 8659 | 7933 8660 | 1410 8661 | 4993 8662 | 1047 8663 | 9469 8664 | 1355 8665 | 2495 8666 | 8975 8667 | 5348 8668 | 997 8669 | 7023 8670 | 2142 8671 | 878 8672 | 6369 8673 | 10452 8674 | 3231 8675 | 9070 8676 | 5078 8677 | 2569 8678 | 9862 8679 | 4199 8680 | 8240 8681 | 8044 8682 | 50 8683 | 7324 8684 | 8868 8685 | 3569 8686 | 8330 8687 | 5459 8688 | 3985 8689 | 4446 8690 | 6829 8691 | 9015 8692 | 10430 8693 | 10102 8694 | 8231 8695 | 1901 8696 | 2839 8697 | 10166 8698 | 611 8699 | 459 8700 | 3478 8701 | 5595 8702 | 4557 8703 | 605 8704 | 9231 8705 | 211 8706 | 5579 8707 | 9887 8708 | 6599 8709 | 3983 8710 | 2178 8711 | 661 8712 | 10143 8713 | 3382 8714 | 4058 8715 | 424 8716 | 9332 8717 | 5363 8718 | 7896 8719 | 3611 8720 | 1130 8721 | 8269 8722 | 56 8723 | 10293 8724 | 3836 8725 | 4487 8726 | 9216 8727 | 4704 8728 | 10162 8729 | 7747 8730 | 6973 8731 | 2087 8732 | 1619 8733 | 494 8734 | 817 8735 | 9890 8736 | 7365 8737 | 4711 8738 | 7445 8739 | 6386 8740 | 5868 8741 | 6935 8742 | 4891 8743 | 7970 8744 | 10090 8745 | 3016 8746 | 6633 8747 | 7046 8748 | 3072 8749 | 5576 8750 | 3429 8751 | 1563 8752 | 9264 8753 | 9308 8754 | 6799 8755 | 3498 8756 | 8175 8757 | 4343 8758 | 2976 8759 | 8806 8760 | 10632 8761 | 5645 8762 | 3044 8763 | 9978 8764 | 3248 8765 | 10552 8766 | 8100 8767 | 10655 8768 | 7721 8769 | 3524 8770 | 2125 8771 | 1210 8772 | 2169 8773 | 5739 8774 | 3162 8775 | 3408 8776 | 10236 8777 | 7776 8778 | 7471 8779 | 682 8780 | 10548 8781 | 3815 8782 | 10439 8783 | 7150 8784 | 2756 8785 | 572 8786 | 5259 8787 | 8549 8788 | 8480 8789 | 2077 8790 | 7476 8791 | 2190 8792 | 4132 8793 | 8187 8794 | 4128 8795 | 7814 8796 | 9661 8797 | 8246 8798 | 2110 8799 | 6871 8800 | 9497 8801 | 8383 8802 | 3889 8803 | 5933 8804 | 3441 8805 | 1838 8806 | 820 8807 | 5869 8808 | 902 8809 | 8358 8810 | 5768 8811 | 9824 8812 | 4232 8813 | 9464 8814 | 6733 8815 | 5604 8816 | 6301 8817 | 2155 8818 | 4455 8819 | 2713 8820 | 894 8821 | 2567 8822 | 9752 8823 | 5673 8824 | 8861 8825 | 9776 8826 | 6585 8827 | 1542 8828 | 2757 8829 | 5585 8830 | 6648 8831 | 2034 8832 | 590 8833 | 3281 8834 | 9182 8835 | 1233 8836 | 4725 8837 | 4573 8838 | 8569 8839 | 5662 8840 | 8234 8841 | 1417 8842 | 393 8843 | 2126 8844 | 5065 8845 | 1698 8846 | 4882 8847 | 9553 8848 | 3871 8849 | 7761 8850 | 3290 8851 | 3100 8852 | 10626 8853 | 7447 8854 | 2275 8855 | 10161 8856 | 2368 8857 | 252 8858 | 10296 8859 | 5887 8860 | 6377 8861 | 2629 8862 | 3692 8863 | 6982 8864 | 2376 8865 | 3082 8866 | 1664 8867 | 8788 8868 | 5981 8869 | 9230 8870 | 186 8871 | 726 8872 | 3391 8873 | 8989 8874 | 6824 8875 | 2766 8876 | 7538 8877 | 9670 8878 | 298 8879 | 4219 8880 | 2404 8881 | 8029 8882 | 5480 8883 | 4416 8884 | 1773 8885 | 8213 8886 | 10104 8887 | 8021 8888 | 4250 8889 | 9232 8890 | 7589 8891 | 1112 8892 | 6881 8893 | 3179 8894 | 8643 8895 | 8369 8896 | 10256 8897 | 1736 8898 | 4214 8899 | 4308 8900 | 7088 8901 | 8495 8902 | 5878 8903 | 1348 8904 | 7746 8905 | 2259 8906 | 5305 8907 | 1985 8908 | 7221 8909 | 6184 8910 | 9977 8911 | 2763 8912 | 1978 8913 | 9172 8914 | 3254 8915 | 1915 8916 | 2600 8917 | 8620 8918 | 8635 8919 | 9603 8920 | 6200 8921 | 7495 8922 | 3238 8923 | 6394 8924 | 1110 8925 | 6094 8926 | 700 8927 | 7551 8928 | 5255 8929 | 8811 8930 | 5731 8931 | 8562 8932 | 4043 8933 | 2584 8934 | 1688 8935 | 10081 8936 | 5174 8937 | 3411 8938 | 10428 8939 | 2343 8940 | 750 8941 | 7076 8942 | 9817 8943 | 6620 8944 | 7692 8945 | 3109 8946 | 9953 8947 | 2336 8948 | 8058 8949 | 7517 8950 | 2572 8951 | 2722 8952 | 10380 8953 | 9924 8954 | 5707 8955 | 9533 8956 | 2535 8957 | 1783 8958 | 7169 8959 | 2648 8960 | 5091 8961 | 5536 8962 | 7053 8963 | 904 8964 | 8985 8965 | 6919 8966 | 7651 8967 | 6482 8968 | 1030 8969 | 4146 8970 | 2269 8971 | 6092 8972 | 191 8973 | 4198 8974 | 1315 8975 | 5530 8976 | 4822 8977 | 7836 8978 | 6332 8979 | 4363 8980 | 7828 8981 | 986 8982 | 6098 8983 | 2171 8984 | 5956 8985 | 3472 8986 | 5550 8987 | 9880 8988 | 4654 8989 | 2436 8990 | 3533 8991 | 2457 8992 | 846 8993 | 7767 8994 | 4310 8995 | 5028 8996 | 4928 8997 | 3734 8998 | 6321 8999 | 4130 9000 | 8656 9001 | 1755 9002 | 8601 9003 | 8854 9004 | 7922 9005 | 5519 9006 | 3899 9007 | 5313 9008 | 1668 9009 | 2672 9010 | 4905 9011 | 5286 9012 | 4953 9013 | 7308 9014 | 4410 9015 | 3691 9016 | 8116 9017 | 6381 9018 | 6480 9019 | 4526 9020 | 6261 9021 | 6567 9022 | 6252 9023 | 5108 9024 | 6227 9025 | 7172 9026 | 7671 9027 | 9136 9028 | 9601 9029 | 6452 9030 | 5177 9031 | 3740 9032 | 9313 9033 | 10211 9034 | 6032 9035 | 8819 9036 | 10101 9037 | 5489 9038 | 8917 9039 | 5416 9040 | 6883 9041 | 1149 9042 | 4969 9043 | 163 9044 | 2875 9045 | 1341 9046 | 3379 9047 | 4803 9048 | 2694 9049 | 7947 9050 | 6509 9051 | 5176 9052 | 5112 9053 | 3434 9054 | 6152 9055 | 5041 9056 | 4254 9057 | 6347 9058 | 10271 9059 | 7783 9060 | 1899 9061 | 3573 9062 | 5136 9063 | 10394 9064 | 7848 9065 | 6008 9066 | 7222 9067 | 3994 9068 | 748 9069 | 8942 9070 | 9099 9071 | 6991 9072 | 2845 9073 | 7526 9074 | 7607 9075 | 2876 9076 | 8563 9077 | 9234 9078 | 4169 9079 | 7244 9080 | 9676 9081 | 714 9082 | 6923 9083 | 8056 9084 | 3724 9085 | 5492 9086 | 3616 9087 | 1029 9088 | 6491 9089 | 4668 9090 | 9146 9091 | 7450 9092 | 1712 9093 | 6720 9094 | 1825 9095 | 6689 9096 | 9563 9097 | 10274 9098 | 7547 9099 | 8625 9100 | 9846 9101 | 1989 9102 | 556 9103 | 3865 9104 | 9363 9105 | 511 9106 | 9596 9107 | 6890 9108 | 2893 9109 | 3222 9110 | 5771 9111 | 8590 9112 | 4068 9113 | 194 9114 | 5470 9115 | 5710 9116 | 9804 9117 | 6851 9118 | 8737 9119 | 10580 9120 | 4150 9121 | 4494 9122 | 728 9123 | 3978 9124 | 1733 9125 | 4774 9126 | 6729 9127 | 3080 9128 | 2340 9129 | 5229 9130 | 5417 9131 | 8913 9132 | 9697 9133 | 4757 9134 | 5847 9135 | 7631 9136 | 4311 9137 | 3278 9138 | 8468 9139 | 2170 9140 | 8084 9141 | 1935 9142 | 8616 9143 | 3975 9144 | 3824 9145 | 2744 9146 | 6554 9147 | 8519 9148 | 10290 9149 | 7820 9150 | 4351 9151 | 8672 9152 | 2907 9153 | 6166 9154 | 6527 9155 | 8466 9156 | 5770 9157 | 7207 9158 | 3376 9159 | 8408 9160 | 6128 9161 | 6683 9162 | 9945 9163 | 3925 9164 | 5957 9165 | 9045 9166 | 5069 9167 | 4752 9168 | 6926 9169 | 10032 9170 | 7215 9171 | 4079 9172 | 809 9173 | 5076 9174 | 10378 9175 | 4509 9176 | 9385 9177 | 1127 9178 | 3898 9179 | 4468 9180 | 6629 9181 | 1016 9182 | 9106 9183 | 9271 9184 | 3959 9185 | 8394 9186 | 10460 9187 | 9583 9188 | 4018 9189 | 7798 9190 | 3823 9191 | 2585 9192 | 9181 9193 | 4894 9194 | 10644 9195 | 7929 9196 | 6928 9197 | 374 9198 | 2377 9199 | 10362 9200 | 3041 9201 | 2191 9202 | 7924 9203 | 2279 9204 | 2483 9205 | 4802 9206 | 773 9207 | 9810 9208 | 8724 9209 | 4726 9210 | 1982 9211 | 1019 9212 | 7311 9213 | 5484 9214 | 3262 9215 | 7881 9216 | 6581 9217 | 4126 9218 | 5129 9219 | 9827 9220 | 6938 9221 | 2556 9222 | 7255 9223 | 9556 9224 | 1343 9225 | 8379 9226 | 2352 9227 | 4820 9228 | 10024 9229 | 9518 9230 | 7449 9231 | 3479 9232 | 6556 9233 | 853 9234 | 7363 9235 | 10041 9236 | 2491 9237 | 5392 9238 | 3984 9239 | 4795 9240 | 6318 9241 | 5227 9242 | 9034 9243 | 518 9244 | 2383 9245 | 2490 9246 | 6869 9247 | 3203 9248 | 5312 9249 | 9927 9250 | 7987 9251 | 4206 9252 | 1265 9253 | 9412 9254 | 1061 9255 | 10482 9256 | 4642 9257 | 2450 9258 | 2449 9259 | 3566 9260 | 6945 9261 | 7961 9262 | 8256 9263 | 10654 9264 | 4459 9265 | 7559 9266 | 2530 9267 | 8844 9268 | 6553 9269 | 1471 9270 | 781 9271 | 3977 9272 | 10596 9273 | 3963 9274 | 34 9275 | 8438 9276 | 5019 9277 | 3225 9278 | 7825 9279 | 8929 9280 | 4358 9281 | 5036 9282 | 2566 9283 | 8268 9284 | 3608 9285 | 7603 9286 | 1378 9287 | 7287 9288 | 7683 9289 | 4674 9290 | 9492 9291 | 6601 9292 | 10503 9293 | 10385 9294 | 2402 9295 | 3891 9296 | 9554 9297 | 7406 9298 | 5337 9299 | 7595 9300 | 6464 9301 | 5055 9302 | 5641 9303 | 7750 9304 | 7514 9305 | 8571 9306 | 10013 9307 | 7819 9308 | 4160 9309 | 5106 9310 | 8079 9311 | 5185 9312 | 2451 9313 | 9597 9314 | 10374 9315 | 10416 9316 | 6963 9317 | 457 9318 | 8448 9319 | 6744 9320 | 180 9321 | 10318 9322 | 3039 9323 | 2031 9324 | 7454 9325 | 8031 9326 | 4671 9327 | 1883 9328 | 7223 9329 | 3868 9330 | 20 9331 | 1666 9332 | 6202 9333 | 190 9334 | 381 9335 | 6352 9336 | 10661 9337 | 2810 9338 | 8823 9339 | 3742 9340 | 2333 9341 | 2836 9342 | 1179 9343 | 7357 9344 | 3313 9345 | 7353 9346 | 4423 9347 | 1980 9348 | 4081 9349 | 2645 9350 | 8164 9351 | 8551 9352 | 2739 9353 | 2158 9354 | 4479 9355 | 5482 9356 | 2538 9357 | 1707 9358 | 1647 9359 | 1395 9360 | 3355 9361 | 1945 9362 | 5849 9363 | 225 9364 | 848 9365 | 6753 9366 | 1772 9367 | 2752 9368 | 5298 9369 | 8692 9370 | 8951 9371 | 2246 9372 | 635 9373 | 519 9374 | 4872 9375 | 7503 9376 | 717 9377 | 3300 9378 | 5717 9379 | 8753 9380 | 8600 9381 | 8032 9382 | 792 9383 | 4830 9384 | 4571 9385 | 391 9386 | 2198 9387 | 788 9388 | 413 9389 | 2679 9390 | 5454 9391 | 232 9392 | 4643 9393 | 5982 9394 | 10289 9395 | 4406 9396 | 89 9397 | 4600 9398 | 5031 9399 | 1639 9400 | 2295 9401 | 8934 9402 | 6610 9403 | 5070 9404 | 9190 9405 | 8474 9406 | 3466 9407 | 811 9408 | 4116 9409 | 2334 9410 | 1805 9411 | 948 9412 | 21 9413 | 4609 9414 | 5456 9415 | 3163 9416 | 6536 9417 | 4436 9418 | 1724 9419 | 1092 9420 | 383 9421 | 9329 9422 | 8387 9423 | 5386 9424 | 523 9425 | 4138 9426 | 6508 9427 | 942 9428 | 9577 9429 | 1726 9430 | 8826 9431 | 9355 9432 | 3647 9433 | 5804 9434 | 1894 9435 | 1412 9436 | 1941 9437 | 8243 9438 | 1423 9439 | 831 9440 | 9073 9441 | 7908 9442 | 107 9443 | 8932 9444 | 123 9445 | 9275 9446 | 6288 9447 | 7718 9448 | 2862 9449 | 4088 9450 | 8735 9451 | 2699 9452 | 8055 9453 | 7230 9454 | 4268 9455 | 10559 9456 | 5327 9457 | 643 9458 | 1167 9459 | 160 9460 | 959 9461 | 8211 9462 | 6738 9463 | 3615 9464 | 1686 9465 | 1631 9466 | 6643 9467 | 4751 9468 | 497 9469 | 7739 9470 | 1477 9471 | 1087 9472 | 7184 9473 | 2266 9474 | 4397 9475 | 5332 9476 | 1440 9477 | 642 9478 | 1540 9479 | 7874 9480 | 3106 9481 | 5429 9482 | 3455 9483 | 6360 9484 | 2930 9485 | 7711 9486 | 3134 9487 | 1523 9488 | 10287 9489 | 8397 9490 | 9695 9491 | 2081 9492 | 6800 9493 | 5527 9494 | 1434 9495 | 5488 9496 | 9161 9497 | 5377 9498 | 2867 9499 | 10185 9500 | 3634 9501 | 639 9502 | 10360 9503 | 7702 9504 | 8509 9505 | 6774 9506 | 5241 9507 | 1399 9508 | 6589 9509 | 8053 9510 | 633 9511 | 1613 9512 | 6814 9513 | 6415 9514 | 6703 9515 | 9520 9516 | 2771 9517 | 2513 9518 | 9743 9519 | 7399 9520 | 349 9521 | 7373 9522 | 9057 9523 | 3877 9524 | 8147 9525 | 3896 9526 | 531 9527 | 6468 9528 | 698 9529 | 6298 9530 | 7159 9531 | 8393 9532 | 2599 9533 | 3737 9534 | 7704 9535 | 2888 9536 | 4484 9537 | 8431 9538 | 106 9539 | 10422 9540 | 2773 9541 | 6904 9542 | 4067 9543 | 362 9544 | 10028 9545 | 9056 9546 | 4077 9547 | 5894 9548 | 4019 9549 | 7497 9550 | 10418 9551 | 4736 9552 | 8573 9553 | 8715 9554 | 409 9555 | 347 9556 | 8364 9557 | 2073 9558 | 7932 9559 | 6506 9560 | 3095 9561 | 4085 9562 | 1617 9563 | 4464 9564 | 3029 9565 | 7978 9566 | 680 9567 | 1610 9568 | 10322 9569 | 6224 9570 | 5362 9571 | 2219 9572 | 1425 9573 | 2218 9574 | 6899 9575 | 934 9576 | 6699 9577 | 8996 9578 | 1703 9579 | 9476 9580 | 4742 9581 | 9223 9582 | 5085 9583 | 8598 9584 | 5130 9585 | 1080 9586 | 7510 9587 | 2165 9588 | 306 9589 | 6302 9590 | 6313 9591 | 9209 9592 | 5667 9593 | 4089 9594 | 1949 9595 | 5149 9596 | 1104 9597 | 3602 9598 | 3973 9599 | 3035 9600 | 7185 9601 | 928 9602 | 2654 9603 | 4117 9604 | 8357 9605 | 2263 9606 | 7734 9607 | 2131 9608 | 9981 9609 | 6322 9610 | 6107 9611 | 4956 9612 | 8096 9613 | 5608 9614 | 10650 9615 | 1384 9616 | 5824 9617 | 4265 9618 | 4686 9619 | 554 9620 | 3030 9621 | 2467 9622 | 3325 9623 | 329 9624 | 5250 9625 | 9430 9626 | 10100 9627 | 2353 9628 | 4031 9629 | 10594 9630 | 3747 9631 | 3728 9632 | 6543 9633 | 8257 9634 | 10495 9635 | 6692 9636 | 10565 9637 | 3857 9638 | 8829 9639 | 3743 9640 | 420 9641 | 909 9642 | 6367 9643 | 2399 9644 | 6632 9645 | 3051 9646 | 5157 9647 | 1793 9648 | 1129 9649 | 8088 9650 | 4315 9651 | 3762 9652 | 10459 9653 | 8297 9654 | 6085 9655 | 2002 9656 | 3008 9657 | 956 9658 | 2113 9659 | 6859 9660 | 8215 9661 | 4267 9662 | 550 9663 | 4918 9664 | 8151 9665 | 5050 9666 | 2562 9667 | 10641 9668 | 51 9669 | 5165 9670 | 9221 9671 | 641 9672 | 58 9673 | 10192 9674 | 10193 9675 | 6701 9676 | 8891 9677 | 3343 9678 | 6823 9679 | 1544 9680 | 3826 9681 | 6366 9682 | 5329 9683 | 359 9684 | 3119 9685 | 10039 9686 | 1446 9687 | 35 9688 | 850 9689 | 8729 9690 | 964 9691 | 2355 9692 | 9444 9693 | 9667 9694 | 2911 9695 | 6521 9696 | 10449 9697 | 7562 9698 | 8896 9699 | 2657 9700 | 7714 9701 | 2942 9702 | 7920 9703 | 6604 9704 | 9783 9705 | 7733 9706 | 2710 9707 | 2800 9708 | 7056 9709 | 9344 9710 | 5867 9711 | 8926 9712 | 6168 9713 | 2518 9714 | 355 9715 | 9662 9716 | 5220 9717 | 5817 9718 | 7355 9719 | 2112 9720 | 1186 9721 | 8779 9722 | 8012 9723 | 7558 9724 | 7797 9725 | 593 9726 | 9899 9727 | 5103 9728 | 7157 9729 | 2807 9730 | 5485 9731 | 3314 9732 | 7832 9733 | 6257 9734 | 697 9735 | 2906 9736 | 6110 9737 | 1975 9738 | 4007 9739 | 7861 9740 | 10134 9741 | 5208 9742 | 7880 9743 | 1995 9744 | 2902 9745 | 1392 9746 | 1199 9747 | 6472 9748 | 1683 9749 | 3397 9750 | 5829 9751 | 6156 9752 | 1246 9753 | 3653 9754 | 8525 9755 | 4644 9756 | 4291 9757 | 8763 9758 | 9976 9759 | 7801 9760 | 8099 9761 | 6524 9762 | 9712 9763 | 10558 9764 | 8310 9765 | 7189 9766 | 8507 9767 | 1419 9768 | 5324 9769 | 10350 9770 | 8349 9771 | 9925 9772 | 9507 9773 | 6243 9774 | 6176 9775 | 9941 9776 | 6376 9777 | 566 9778 | 6786 9779 | 5628 9780 | 2723 9781 | 8646 9782 | 3604 9783 | 1430 9784 | 8977 9785 | 7925 9786 | 7955 9787 | 10358 9788 | 2422 9789 | 8235 9790 | 2432 9791 | 5766 9792 | 10457 9793 | 9636 9794 | 3741 9795 | 10011 9796 | 972 9797 | 2959 9798 | 10512 9799 | 5991 9800 | 7991 9801 | 9107 9802 | 6078 9803 | 4334 9804 | 7080 9805 | 9001 9806 | 8774 9807 | 4183 9808 | 5146 9809 | 8816 9810 | 318 9811 | 4433 9812 | 7249 9813 | 8677 9814 | 8047 9815 | 182 9816 | 8343 9817 | 1196 9818 | 2628 9819 | 8377 9820 | 6409 9821 | 6040 9822 | 10000 9823 | 9745 9824 | 1947 9825 | 3683 9826 | 5266 9827 | 3591 9828 | 9883 9829 | 4412 9830 | 597 9831 | 1155 9832 | 5515 9833 | 9273 9834 | 258 9835 | 576 9836 | 9947 9837 | 5359 9838 | 3882 9839 | 7384 9840 | 7912 9841 | 3188 9842 | 3367 9843 | 6656 9844 | 5381 9845 | 8524 9846 | 1307 9847 | 9011 9848 | 6292 9849 | 4695 9850 | 9416 9851 | 220 9852 | 10076 9853 | 6348 9854 | 9751 9855 | 1212 9856 | 10365 9857 | 3916 9858 | 2877 9859 | 321 9860 | 6070 9861 | 4026 9862 | 6572 9863 | 5800 9864 | 5452 9865 | 7282 9866 | 237 9867 | 9738 9868 | 1579 9869 | 7368 9870 | 2028 9871 | 6188 9872 | 7854 9873 | 2655 9874 | 3428 9875 | 3246 9876 | 1070 9877 | 7618 9878 | 471 9879 | 8390 9880 | 5308 9881 | 4284 9882 | 9246 9883 | 8508 9884 | 405 9885 | 1904 9886 | 5815 9887 | 3709 9888 | 8666 9889 | 7307 9890 | 6687 9891 | 1370 9892 | 3282 9893 | 9188 9894 | 7572 9895 | 4015 9896 | 3495 9897 | 1009 9898 | 9548 9899 | 612 9900 | 9881 9901 | 7905 9902 | 6165 9903 | 8015 9904 | 2313 9905 | 3853 9906 | 8009 9907 | 9041 9908 | 3445 9909 | 6674 9910 | 9786 9911 | 8282 9912 | 4020 9913 | 5672 9914 | 990 9915 | 9874 9916 | 9566 9917 | 7575 9918 | 8644 9919 | 6337 9920 | 6884 9921 | 4127 9922 | 2418 9923 | 1662 9924 | 9083 9925 | 8727 9926 | 2175 9927 | 378 9928 | 854 9929 | 8881 9930 | 707 9931 | 10640 9932 | 5633 9933 | 2516 9934 | 2117 9935 | 8419 9936 | 7012 9937 | 4942 9938 | 5844 9939 | 9754 9940 | 10181 9941 | 9870 9942 | 1538 9943 | 6921 9944 | 5855 9945 | 552 9946 | 5842 9947 | 8375 9948 | 7552 9949 | 3678 9950 | 7164 9951 | 6006 9952 | 6627 9953 | 3485 9954 | 5928 9955 | 4852 9956 | 8103 9957 | 7242 9958 | 2277 9959 | 9581 9960 | 6111 9961 | 4917 9962 | 4070 9963 | 2550 9964 | 2035 9965 | 2653 9966 | 4005 9967 | 2915 9968 | 5099 9969 | 2986 9970 | 6803 9971 | 1887 9972 | 9088 9973 | 9335 9974 | 7152 9975 | 3026 9976 | 9064 9977 | 8714 9978 | 1797 9979 | 9996 9980 | 4543 9981 | 1275 9982 | 6059 9983 | 9098 9984 | 6244 9985 | 1743 9986 | 2997 9987 | 7939 9988 | 2598 9989 | 3482 9990 | 8373 9991 | 521 9992 | 4325 9993 | 4542 9994 | 4090 9995 | 9623 9996 | 969 9997 | 4706 9998 | 5773 9999 | 10324 10000 | 9770 10001 | 9756 10002 | 9744 10003 | 8733 10004 | 1004 10005 | 3817 10006 | 3807 10007 | 4943 10008 | 9173 10009 | 4570 10010 | 9952 10011 | 3803 10012 | 8174 10013 | 9557 10014 | 7340 10015 | 9784 10016 | 6388 10017 | 7186 10018 | 9541 10019 | 3774 10020 | 753 10021 | 6764 10022 | 4923 10023 | 2332 10024 | 4940 10025 | 1589 10026 | 3175 10027 | 7432 10028 | 5685 10029 | 8072 10030 | 3993 10031 | 4765 10032 | 2173 10033 | 1923 10034 | 3593 10035 | 2908 10036 | 7653 10037 | 2765 10038 | 542 10039 | 6185 10040 | 7537 10041 | 8217 10042 | 8807 10043 | 4710 10044 | 6930 10045 | 1360 10046 | 4559 10047 | 8516 10048 | 6046 10049 | 450 10050 | 1693 10051 | 4444 10052 | 176 10053 | 5479 10054 | 9274 10055 | 1856 10056 | 8856 10057 | 5754 10058 | 10419 10059 | 895 10060 | 9891 10061 | 9217 10062 | 4297 10063 | 6947 10064 | 7200 10065 | 3610 10066 | 5458 10067 | 5728 10068 | 1200 10069 | 407 10070 | 6290 10071 | 7453 10072 | 4633 10073 | 3681 10074 | 8821 10075 | 6688 10076 | 10058 10077 | 4269 10078 | 3579 10079 | 6248 10080 | 3374 10081 | 5818 10082 | 6658 10083 | 8770 10084 | 9979 10085 | 4450 10086 | 9242 10087 | 10073 10088 | 965 10089 | 8422 10090 | 2016 10091 | 9708 10092 | 2981 10093 | 6889 10094 | 1722 10095 | 5039 10096 | 7856 10097 | 4041 10098 | 1068 10099 | 6401 10100 | 1037 10101 | 4963 10102 | 6671 10103 | 9747 10104 | 6967 10105 | 5095 10106 | 5900 10107 | 2666 10108 | 4480 10109 | 3666 10110 | 9923 10111 | 9841 10112 | 240 10113 | 10522 10114 | 716 10115 | 2977 10116 | 8097 10117 | 3200 10118 | 1893 10119 | 1536 10120 | 3544 10121 | 4497 10122 | 1448 10123 | 2725 10124 | 10085 10125 | 10012 10126 | 7166 10127 | 10582 10128 | 2519 10129 | 8860 10130 | 7983 10131 | 4186 10132 | 7452 10133 | 4569 10134 | 586 10135 | 3020 10136 | 3473 10137 | 857 10138 | 2664 10139 | 5826 10140 | 9321 10141 | 7305 10142 | 6382 10143 | 1214 10144 | 5872 10145 | 10009 10146 | 4382 10147 | 6351 10148 | 4619 10149 | 1389 10150 | 2852 10151 | 6123 10152 | 5075 10153 | 4927 10154 | 1981 10155 | 7745 10156 | 5507 10157 | 1551 10158 | 9822 10159 | 7587 10160 | 3779 10161 | 6406 10162 | 609 10163 | 9342 10164 | 5156 10165 | 1121 10166 | 8456 10167 | 482 10168 | 2049 10169 | 10316 10170 | 9820 10171 | 277 10172 | 6804 10173 | 3221 10174 | 4575 10175 | 9495 10176 | 4282 10177 | 10132 10178 | 9753 10179 | 9473 10180 | 1304 10181 | 6410 10182 | 2809 10183 | 5816 10184 | 7668 10185 | 8632 10186 | 8723 10187 | 9833 10188 | 10607 10189 | 6499 10190 | 9433 10191 | 5476 10192 | 5503 10193 | 1535 10194 | 6512 10195 | 9391 10196 | 9302 10197 | 9627 10198 | 8082 10199 | 9868 10200 | 6621 10201 | 10112 10202 | 652 10203 | 342 10204 | 219 10205 | 5190 10206 | 9780 10207 | 8159 10208 | 4507 10209 | 7111 10210 | 5577 10211 | 5401 10212 | 1509 10213 | 2693 10214 | 8023 10215 | 1520 10216 | 6286 10217 | 6885 10218 | 6637 10219 | 5042 10220 | 577 10221 | 8167 10222 | 1616 10223 | 8125 10224 | 3463 10225 | 8741 10226 | 3070 10227 | 9663 10228 | 6965 10229 | 8220 10230 | 4439 10231 | 1900 10232 | 408 10233 | 8170 10234 | 3105 10235 | 7664 10236 | 5187 10237 | 4551 10238 | 6210 10239 | 7666 10240 | 6238 10241 | 3670 10242 | 8653 10243 | 10272 10244 | 10174 10245 | 5533 10246 | 5450 10247 | 3519 10248 | 1869 10249 | 3265 10250 | 4797 10251 | 6485 10252 | 7203 10253 | 5244 10254 | 551 10255 | 1506 10256 | 4807 10257 | 941 10258 | 3122 10259 | 9472 10260 | 4950 10261 | 7330 10262 | 963 10263 | 9643 10264 | 2374 10265 | 6169 10266 | 7900 10267 | 439 10268 | 7859 10269 | 7842 10270 | 3084 10271 | 5269 10272 | 7726 10273 | 1763 10274 | 8884 10275 | 8197 10276 | 5015 10277 | 10592 10278 | 4293 10279 | 4493 10280 | 2461 10281 | 10306 10282 | 9477 10283 | 3350 10284 | 3672 10285 | 5517 10286 | 7153 10287 | 8988 10288 | 1436 10289 | 5472 10290 | 4226 10291 | 7294 10292 | 10382 10293 | 10218 10294 | 7481 10295 | 1363 10296 | 412 10297 | 9175 10298 | 8757 10299 | 6163 10300 | 4676 10301 | 8518 10302 | 1099 10303 | 4518 10304 | 1657 10305 | 2947 10306 | 10611 10307 | 9882 10308 | 7151 10309 | 5382 10310 | 539 10311 | 3703 10312 | 3987 10313 | 8403 10314 | 6755 10315 | 5356 10316 | 5892 10317 | 2861 10318 | 296 10319 | 2626 10320 | 352 10321 | 1807 10322 | 7772 10323 | 2738 10324 | 2650 10325 | 345 10326 | 7143 10327 | 10573 10328 | 7524 10329 | 794 10330 | 2952 10331 | 4640 10332 | 8114 10333 | 10148 10334 | 896 10335 | 5904 10336 | 9035 10337 | 1674 10338 | 8335 10339 | 3887 10340 | 1855 10341 | 9395 10342 | 9368 10343 | 10187 10344 | 1690 10345 | 6955 10346 | 8189 10347 | 7014 10348 | 8207 10349 | 8218 10350 | 1984 10351 | 9713 10352 | 8065 10353 | 9748 10354 | 1280 10355 | 9369 10356 | 732 10357 | 8185 10358 | 3017 10359 | 8143 10360 | 9588 10361 | 9615 10362 | 3232 10363 | 3393 10364 | 1300 10365 | 3650 10366 | 5819 10367 | 3192 10368 | 8750 10369 | 515 10370 | 7981 10371 | 5758 10372 | 8606 10373 | 3263 10374 | 6547 10375 | 3368 10376 | 8117 10377 | 2842 10378 | 5532 10379 | 583 10380 | 3252 10381 | 9764 10382 | 3261 10383 | 2261 10384 | 1114 10385 | 7578 10386 | 855 10387 | 7998 10388 | 4627 10389 | 5418 10390 | 9461 10391 | 7063 10392 | 7119 10393 | 9155 10394 | 8425 10395 | 1586 10396 | 3659 10397 | 10577 10398 | 1409 10399 | 5231 10400 | 6255 10401 | 5715 10402 | 2674 10403 | 10639 10404 | 5481 10405 | 940 10406 | 3453 10407 | 3010 10408 | 8306 10409 | 2776 10410 | 7716 10411 | 951 10412 | 9044 10413 | 8585 10414 | 4305 10415 | 8208 10416 | 7271 10417 | 7573 10418 | 3259 10419 | 9788 10420 | 8121 10421 | 1456 10422 | 7897 10423 | 7996 10424 | 10297 10425 | 9115 10426 | 5921 10427 | 6797 10428 | 4799 10429 | 10292 10430 | 7701 10431 | 3770 10432 | 9755 10433 | 3599 10434 | 8067 10435 | 8564 10436 | 4620 10437 | 5572 10438 | 9407 10439 | 3874 10440 | 2388 10441 | 2473 10442 | 8483 10443 | 8503 10444 | 6887 10445 | 8756 10446 | 7509 10447 | 5409 10448 | 4180 10449 | 2587 10450 | 7829 10451 | 307 10452 | 7148 10453 | 9532 10454 | 8802 10455 | 9514 10456 | 358 10457 | 7882 10458 | 3404 10459 | 7642 10460 | 7796 10461 | 1428 10462 | 3414 10463 | 9229 10464 | 3581 10465 | 3438 10466 | 8946 10467 | 4403 10468 | 3270 10469 | 3583 10470 | 390 10471 | 2448 10472 | 9791 10473 | 5342 10474 | 1839 10475 | 10505 10476 | 5200 10477 | 984 10478 | 3061 10479 | 6940 10480 | 4798 10481 | 8452 10482 | 6436 10483 | 7893 10484 | 2341 10485 | 1091 10486 | 10468 10487 | 370 10488 | 7698 10489 | 6995 10490 | 3974 10491 | 9774 10492 | 5837 10493 | 1778 10494 | 1823 10495 | 5126 10496 | 7137 10497 | 10417 10498 | 8501 10499 | 7314 10500 | 616 10501 | 3567 10502 | 7773 10503 | 3578 10504 | 3104 10505 | 9715 10506 | 9538 10507 | 3980 10508 | 10525 10509 | 8667 10510 | 184 10511 | 9474 10512 | 9023 10513 | 368 10514 | 3906 10515 | 10033 10516 | 9399 10517 | 4659 10518 | 9362 10519 | 3073 10520 | 7266 10521 | 9438 10522 | 3917 10523 | 7378 10524 | 4326 10525 | 9973 10526 | 2088 10527 | 3962 10528 | 7411 10529 | 2968 10530 | 5022 10531 | 3912 10532 | 4678 10533 | 4829 10534 | 6141 10535 | 351 10536 | 4961 10537 | 4545 10538 | 10330 10539 | 4857 10540 | 3130 10541 | 2493 10542 | 4625 10543 | 3972 10544 | 4245 10545 | 541 10546 | 3417 10547 | 7372 10548 | 4915 10549 | 2652 10550 | 9305 10551 | 7350 10552 | 7224 10553 | 5951 10554 | 344 10555 | 3708 10556 | 845 10557 | 9885 10558 | 6570 10559 | 6263 10560 | 2271 10561 | 2707 10562 | 3538 10563 | 7728 10564 | 1728 10565 | 6714 10566 | 1556 10567 | 4255 10568 | 8864 10569 | 432 10570 | 9616 10571 | 834 10572 | 6475 10573 | 8365 10574 | 4506 10575 | 4035 10576 | 1243 10577 | 6378 10578 | 4592 10579 | 9772 10580 | 9397 10581 | 3185 10582 | 9671 10583 | 5033 10584 | 9082 10585 | 7123 10586 | 10221 10587 | 10527 10588 | 1700 10589 | 1357 10590 | 7892 10591 | 7884 10592 | 3090 10593 | 1574 10594 | 93 10595 | 6219 10596 | 4261 10597 | 4766 10598 | 4215 10599 | 5487 10600 | 328 10601 | 5104 10602 | 6906 10603 | 84 10604 | 9795 10605 | 3442 10606 | 2344 10607 | 9400 10608 | 8317 10609 | 4749 10610 | 357 10611 | 6504 10612 | 5512 10613 | 9624 10614 | 4383 10615 | 1124 10616 | 1867 10617 | 2072 10618 | 1444 10619 | 9281 10620 | 3427 10621 | 4739 10622 | 7407 10623 | 3205 10624 | 1910 10625 | 567 10626 | 10066 10627 | 2447 10628 | 9705 10629 | 4147 10630 | 3155 10631 | 10026 10632 | 4925 10633 | 8959 10634 | 9092 10635 | 5441 10636 | 2054 10637 | 6757 10638 | 3211 10639 | 5990 10640 | 1495 10641 | 8328 10642 | 7120 10643 | 4836 10644 | 3509 10645 | 1753 10646 | 7817 10647 | 537 10648 | 8127 10649 | 1548 10650 | 7155 10651 | 767 10652 | 1784 10653 | 3520 10654 | 6728 10655 | 2017 10656 | 7545 10657 | 5263 10658 | 6878 10659 | 1303 10660 | 5886 10661 | 4321 10662 | 1148 10663 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.16 2 | -------------------------------------------------------------------------------- /project/site.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "4.0.0") 2 | -------------------------------------------------------------------------------- /run_scripts/train_ac_blstm.sh: -------------------------------------------------------------------------------- 1 | ROOT=$(cd "$(dirname $0)/.."; pwd) 2 | 3 | # put yur mxnet jar file in the lib folder 4 | MXNET_JAR_FILE=$ROOT/lib/mxnet-full_2.11-INTERNAL.jar 5 | 6 | CLASS_PATH=$MXNET_JAR_FILE:$ROOT/target/scala-2.11/classes:$HOME/.ivy2/cache/org.scala-lang/scala-library/jars/scala-library-2.11.8.jar:\ 7 | $HOME/.ivy2/cache/args4j/args4j/bundles/args4j-2.33.jar:\ 8 | $HOME/.ivy2/cache/org.slf4j/slf4j-api/jars/slf4j-api-1.7.25.jar 9 | 10 | # to do 10-fold cross validation experiments, range from 0 ~ 9 11 | CROSS_VALIDATION_ID=0 12 | 13 | # -1 for cpu 14 | GPU=0 15 | 16 | # learning rate, default 0.0001 17 | LEARNING_RATE=0.0001 18 | 19 | BATCH_SIZE=50 20 | 21 | SAVE_MODRL_PATH=$ROOT/datas/trainModels 22 | 23 | if [ ! -d $SAVE_MODRL_PATH ] ; then 24 | mkdir -p $SAVE_MODRL_PATH 25 | fi 26 | 27 | MR_DATA_PATH=$ROOT/datas/MR 28 | 29 | # pretrained word2vec file path 30 | WORD2VEC_FILE_PATH=$ROOT/datas/GoogleNews-vectors-negative300.bin 31 | # pretrained word2vec file format, bin -> 1, text -> 0 32 | FORMAT=1 33 | 34 | java -Xmx29G -cp $CLASS_PATH \ 35 | experiments.AC_BLSTM_TextClassification \ 36 | --cross-validation-id $CROSS_VALIDATION_ID \ 37 | --lr $LEARNING_RATE \ 38 | --gpu $GPU \ 39 | --mr-dataset-path $MR_DATA_PATH \ 40 | --w2v-file-path $WORD2VEC_FILE_PATH \ 41 | --w2v-format-bin $FORMAT \ 42 | --batch-size $BATCH_SIZE \ 43 | --save-model-path $SAVE_MODRL_PATH 44 | -------------------------------------------------------------------------------- /run_scripts/train_g_ac_blstm.sh: -------------------------------------------------------------------------------- 1 | ROOT=$(cd "$(dirname $0)/.."; pwd) 2 | 3 | # put yur mxnet jar file in the lib folder 4 | MXNET_JAR_FILE=$ROOT/lib/mxnet-full_2.11-INTERNAL.jar 5 | 6 | CLASS_PATH=$MXNET_JAR_FILE:$ROOT/target/scala-2.11/classes:$HOME/.ivy2/cache/org.scala-lang/scala-library/jars/scala-library-2.11.8.jar:\ 7 | $HOME/.ivy2/cache/args4j/args4j/bundles/args4j-2.33.jar:\ 8 | $HOME/.ivy2/cache/org.slf4j/slf4j-api/jars/slf4j-api-1.7.25.jar 9 | 10 | # to do 10-fold cross validation experiments, range from 0 ~ 9 11 | CROSS_VALIDATION_ID=1 12 | 13 | # -1 for cpu 14 | GPU=0 15 | 16 | # learning rate, default 0.0001 17 | LEARNING_RATE=0.0001 18 | 19 | BATCH_SIZE=50 20 | 21 | # G_BATCH = BATCH_SIZE * 0.2 22 | G_BATCH=10 23 | 24 | 25 | SAVE_MODRL_PATH=$ROOT/datas/trainGModels 26 | 27 | if [ ! -d $SAVE_MODRL_PATH ] ; then 28 | mkdir -p $SAVE_MODRL_PATH 29 | fi 30 | 31 | MR_DATA_PATH=$ROOT/datas/MR 32 | 33 | # pretrained word2vec file path 34 | WORD2VEC_FILE_PATH=$ROOT/datas/GoogleNews-vectors-negative300.bin 35 | # pretrained word2vec file format, bin -> 1, text -> 0 36 | FORMAT=1 37 | 38 | java -Xmx29G -cp $CLASS_PATH \ 39 | experiments.G_AC_BLSTM \ 40 | --cross-validation-id $CROSS_VALIDATION_ID \ 41 | --lr $LEARNING_RATE \ 42 | --gpu $GPU \ 43 | --mr-dataset-path $MR_DATA_PATH \ 44 | --w2v-file-path $WORD2VEC_FILE_PATH \ 45 | --w2v-format-bin $FORMAT \ 46 | --batch-size $BATCH_SIZE \ 47 | --gan-batch $G_BATCH \ 48 | --save-model-path $SAVE_MODRL_PATH 49 | -------------------------------------------------------------------------------- /src/main/scala/experiments/AC_BLSTM_TextClassification.scala: -------------------------------------------------------------------------------- 1 | package experiments 2 | 3 | import org.kohsuke.args4j.{CmdLineParser, Option} 4 | import org.slf4j.LoggerFactory 5 | import scala.collection.JavaConverters._ 6 | import org.apache.mxnet.Initializer 7 | import org.apache.mxnet.Uniform 8 | import org.apache.mxnet.Context 9 | import org.apache.mxnet.Symbol 10 | import org.apache.mxnet.Shape 11 | import org.apache.mxnet.NDArray 12 | import org.apache.mxnet.Executor 13 | import org.apache.mxnet.optimizer.RMSProp 14 | import org.apache.mxnet.Optimizer 15 | import org.apache.mxnet.Model 16 | 17 | import org.apache.mxnet.Xavier 18 | import org.apache.mxnet.optimizer.AdaDelta 19 | import org.apache.mxnet.optimizer.Adam 20 | 21 | import scala.util.Random 22 | import java.io.PrintWriter 23 | import scala.io.Source 24 | import scala.collection.mutable.ArrayBuffer 25 | import org.apache.mxnet.util.OptionConversion._ 26 | 27 | /** 28 | * Implementation of the paper 29 | * AC-BLSTM: Asymmetric Convolutional Bidirectional LSTM Networks for Text Classification 30 | * 31 | * @author Depeng Liang 32 | */ 33 | object AC_BLSTM_TextClassification { 34 | 35 | private val logger = LoggerFactory.getLogger(classOf[AC_BLSTM_TextClassification]) 36 | 37 | case class CNNModel(cnnExec: Executor, symbol: Symbol, data: NDArray, label: NDArray, 38 | argsDict: Map[String, NDArray], gradDict: Map[String, NDArray]) 39 | 40 | final case class LSTMState(c: Symbol, h: Symbol) 41 | final case class LSTMParam(i2hWeight: Symbol, i2hBias: Symbol, 42 | h2hWeight: Symbol, h2hBias: Symbol) 43 | 44 | // LSTM Cell symbol 45 | def lstm( 46 | numHidden: Int, 47 | inData: Symbol, 48 | prevState: LSTMState, 49 | param: LSTMParam, 50 | seqIdx: Int, 51 | layerIdx: Int, 52 | dropout: Float = 0f): LSTMState = { 53 | 54 | val inDataa = { 55 | if (dropout > 0f) 56 | Symbol.api.Dropout(inData, dropout) 57 | else inData 58 | } 59 | 60 | val i2h = Symbol.api.FullyConnected(inDataa, param.i2hWeight, param.i2hBias, numHidden * 4, name = s"t${seqIdx}_l${layerIdx}_i2h") 61 | 62 | val h2h = Symbol.api.FullyConnected(prevState.h, param.h2hWeight, param.h2hBias, numHidden * 4, name = s"t${seqIdx}_l${layerIdx}_h2h") 63 | 64 | val gates = i2h + h2h 65 | 66 | val sliceGates = Symbol.api.SliceChannel(gates, num_outputs = 4, name = s"t${seqIdx}_l${layerIdx}_slice") 67 | 68 | val ingate = Symbol.api.sigmoid(sliceGates.get(0)) 69 | val inTransform = Symbol.api.tanh(sliceGates.get(1)) 70 | val forgetGate = Symbol.api.sigmoid(sliceGates.get(2)) 71 | val outGate = Symbol.api.sigmoid(sliceGates.get(3)) 72 | val nextC = (forgetGate * prevState.c) + (ingate * inTransform) 73 | val nextH = outGate * Symbol.api.tanh(nextC) 74 | LSTMState(c = nextC, h = nextH) 75 | } 76 | 77 | def makeTextCNN(sentenceSize: Int, numEmbed: Int, batchSize: Int, numHidden: Int, numLstmLayer: Int = 1, 78 | numLabel: Int = 2, filterList: Array[Int] = Array(3, 4, 5), numFilter: Int = 100, dropout: Float = 0.5f): Symbol = { 79 | 80 | var inputX = Symbol.Variable("data") 81 | val inputY = Symbol.Variable("softmax_label") 82 | 83 | // add dropout before conv layers 84 | if (dropout > 0f) inputX = Symbol.api.Dropout(inputX, dropout) 85 | 86 | val newNumFilter = numFilter 87 | val totalDim = newNumFilter * filterList.length 88 | var seqLen = sentenceSize - filterList.sorted.reverse.head + 1 89 | 90 | val windowSeqOutputs = filterList.map { filterSize => 91 | // inception v4 2 92 | var conv = Symbol.api.Convolution(inputX, kernel = Shape(1, numEmbed), num_filter = numFilter, cudnn_off = true) 93 | var bn = Symbol.api.BatchNorm(conv) 94 | var relu = Symbol.api.relu(bn) 95 | 96 | val len = sentenceSize - filterSize + 1 97 | 98 | conv = Symbol.api.Convolution(relu, kernel = Shape(filterSize, 1), num_filter = numFilter, cudnn_off = true, dilate = Shape(1, 1)) 99 | bn = Symbol.api.BatchNorm(conv) 100 | relu = Symbol.api.relu(bn) 101 | 102 | if (len > seqLen) { 103 | val partOne = Symbol.api.slice_axis(relu, axis = 2, begin = 0, end = (seqLen - 1)) 104 | var partTwo = Symbol.api.slice_axis(relu, axis = 2, begin = (seqLen - 1), end = len) 105 | partTwo = Symbol.api.Flatten(partTwo) 106 | partTwo = Symbol.api.FullyConnected(partTwo, num_hidden = newNumFilter) 107 | partTwo = Symbol.api.Reshape(partTwo, target_shape = Shape(batchSize, numFilter, 1, 1)) 108 | Symbol.api.Concat(Array(partOne, partTwo), num_args = 2, dim = 2) 109 | } else relu 110 | } 111 | 112 | val lstmInputs = { 113 | val concats = Symbol.api.Concat(windowSeqOutputs, num_args = windowSeqOutputs.length, dim = 1) 114 | Symbol.api.SliceChannel(concats, axis = 2, num_outputs = seqLen, squeeze_axis = true) 115 | } 116 | 117 | // bi-lstm 118 | var forwardParamCells = Array[LSTMParam]() 119 | var forwardLastStates = Array[LSTMState]() 120 | for (i <- 0 until numLstmLayer) { 121 | forwardParamCells = forwardParamCells :+ LSTMParam(i2hWeight = Symbol.Variable(s"f_l${i}_i2h_weight"), 122 | i2hBias = Symbol.Variable(s"f_l${i}_i2h_bias"), 123 | h2hWeight = Symbol.Variable(s"f_l${i}_h2h_weight"), 124 | h2hBias = Symbol.Variable(s"f_l${i}_h2h_bias")) 125 | forwardLastStates = forwardLastStates :+ LSTMState(c = Symbol.Variable(s"f_l${i}_init_c"), 126 | h = Symbol.Variable(s"f_l${i}_init_h")) 127 | } 128 | assert(forwardLastStates.length == numLstmLayer) 129 | 130 | var backwardParamCells = Array[LSTMParam]() 131 | var backwardLastStates = Array[LSTMState]() 132 | for (i <- 0 until numLstmLayer) { 133 | backwardParamCells = backwardParamCells :+ LSTMParam(i2hWeight = Symbol.Variable(s"b_l${i}_i2h_weight"), 134 | i2hBias = Symbol.Variable(s"b_l${i}_i2h_bias"), 135 | h2hWeight = Symbol.Variable(s"b_l${i}_h2h_weight"), 136 | h2hBias = Symbol.Variable(s"b_l${i}_h2h_bias")) 137 | backwardLastStates = backwardLastStates :+ LSTMState(c = Symbol.Variable(s"b_l${i}_init_c"), 138 | h = Symbol.Variable(s"b_l${i}_init_h")) 139 | } 140 | assert(backwardLastStates.length == numLstmLayer) 141 | 142 | // forward 143 | var forwardHiddenAll = Array[Symbol]() 144 | var dpRatio = 0f 145 | var hidden: Symbol = null 146 | for (seqIdx <- 0 until seqLen) { 147 | hidden = lstmInputs.get(seqIdx) 148 | // stack LSTM 149 | for (i <- 0 until numLstmLayer) { 150 | if (i == 0) dpRatio = 0f else dpRatio = dropout 151 | val nextState = lstm(numHidden, inData = hidden, 152 | prevState = forwardLastStates(i), 153 | param = forwardParamCells(i), 154 | seqIdx = seqIdx, layerIdx = i, dropout = dpRatio) 155 | hidden = nextState.h 156 | forwardLastStates(i) = nextState 157 | } 158 | // add dropout before softmax 159 | if (dropout > 0f) hidden = Symbol.api.Dropout(hidden, p = dropout) 160 | forwardHiddenAll = forwardHiddenAll :+ hidden 161 | } 162 | 163 | // backward 164 | var badkwardHiddenAll = Array[Symbol]() 165 | dpRatio = 0f 166 | for (seqIdx <- 0 until seqLen) { 167 | val k = seqLen - seqIdx - 1 168 | hidden = lstmInputs.get(k) 169 | // stack LSTM 170 | for (i <- 0 until numLstmLayer) { 171 | if (i == 0) dpRatio = 0f else dpRatio = dropout 172 | val nextState = lstm(numHidden, inData = hidden, 173 | prevState = backwardLastStates(i), 174 | param = backwardParamCells(i), 175 | seqIdx = k, layerIdx = i, dropout = dpRatio) 176 | hidden = nextState.h 177 | backwardLastStates(i) = nextState 178 | } 179 | // add dropout before softmax 180 | if (dropout > 0f) hidden = Symbol.api.Dropout(hidden, p = dropout) 181 | badkwardHiddenAll = hidden +: badkwardHiddenAll 182 | } 183 | 184 | val syms = forwardHiddenAll.zip(badkwardHiddenAll).map { case (f, b) => 185 | var tmp = Symbol.api.Concat(Array(f, b), num_args = 2, dim = 1) 186 | Symbol.api.Dropout(tmp, p = 0.5f) 187 | } 188 | 189 | var hiddenConcat = Symbol.api.Concat(syms, num_args = syms.length, dim = 1) 190 | val fc = Symbol.api.FullyConnected(hiddenConcat, num_hidden = numLabel) 191 | val sm = Symbol.api.SoftmaxOutput(fc, label = inputY) 192 | sm 193 | } 194 | 195 | def setupCnnModel(ctx: Context, batchSize: Int, sentenceSize: Int, numEmbed: Int, numHidden: Int = 100, 196 | numLstmLayer: Int = 1, inputChannels: Int = 1, numLabel: Int = 2, numFilter: Int = 100, filterList: Array[Int ] = Array(2, 3, 4), 197 | dropout: Float = 0.5f, initializer: Initializer = new Uniform(0.1f), resumeModelPath: String = null): CNNModel = { 198 | 199 | val cnn = makeTextCNN(sentenceSize, numEmbed, batchSize, numHidden, numLstmLayer, numLabel, filterList, numFilter, dropout) 200 | val argNames = cnn.listArguments() 201 | val auxNames = cnn.listAuxiliaryStates() 202 | 203 | // try bi-lstm 204 | val forwardInitC = for (l <- 0 until numLstmLayer) yield (s"f_l${l}_init_c", (batchSize, numHidden)) 205 | val fordwardInitH = for (l <- 0 until numLstmLayer) yield (s"f_l${l}_init_h", (batchSize, numHidden)) 206 | val backwardInitC = for (l <- 0 until numLstmLayer) yield (s"b_l${l}_init_c", (batchSize, numHidden)) 207 | val backwardInitH = for (l <- 0 until numLstmLayer) yield (s"b_l${l}_init_h", (batchSize, numHidden)) 208 | val initStates = (forwardInitC ++ fordwardInitH ++ backwardInitC ++ backwardInitH).map(x => x._1 -> Shape(x._2._1, x._2._2)).toMap 209 | 210 | val dataShapes = Map("data" -> Shape(batchSize, inputChannels, sentenceSize, numEmbed)) ++ initStates 211 | 212 | val (argShapes, outShapes, auxShapes) = cnn.inferShape(dataShapes) 213 | 214 | val argsDict = argNames.zip(argShapes).map { case (name, shape) => 215 | val nda = NDArray.zeros(shape, ctx) 216 | if (!dataShapes.contains(name) && name != "softmax_label") { 217 | initializer(name, nda) 218 | } 219 | name -> nda }.toMap 220 | 221 | argsDict.foreach { case (name, array) => println(s"$name, ${array.shape}") } 222 | 223 | val argsGradDict = argNames.zip(argShapes) 224 | .filter(x => x._1 != "softmax_label" && x._1 != "data") 225 | .map { x => 226 | val nda = NDArray.zeros(x._2, ctx) 227 | x._1 -> nda}.toMap 228 | 229 | if (resumeModelPath != null) { 230 | var pretrained = NDArray.load2Map(resumeModelPath) 231 | argsDict.foreach { case (name, ndA) => 232 | if (name != "softmax_label" && !dataShapes.contains(name)) { 233 | val key = s"arg:$name" 234 | if(pretrained.contains(key)) ndA.set(pretrained(key)) 235 | else println(s"Skip argument $name") 236 | pretrained(key).dispose() 237 | } 238 | } 239 | pretrained = null 240 | } 241 | 242 | val auxDict = auxNames.zip(auxShapes.map(NDArray.zeros(_, ctx))).toMap 243 | val cnnExec = cnn.bind(ctx, argsDict, argsGradDict, "write", auxDict, null, null) 244 | 245 | val data = argsDict("data") 246 | val label = argsDict("softmax_label") 247 | CNNModel(cnnExec, cnn, data, label, argsDict, argsGradDict) 248 | } 249 | 250 | def trainCNN(model: CNNModel, trainBatches: Array[Array[Array[Float]]], 251 | trainLabels: Array[Float], devBatches: Array[Array[Array[Float]]], 252 | devLabels: Array[Float], batchSize: Int, saveModelPath: String, 253 | learningRate: Float = 0.001f): Unit = { 254 | val maxGradNorm = 0.5f 255 | val epoch = 1000 256 | val opt = new RMSProp(learningRate = learningRate, wd = 0.001f) 257 | var start = 0L 258 | var end = 0L 259 | var numCorrect = 0f 260 | var numTotal = 0f 261 | var factor = 0.5f 262 | var maxAccuracy = -1f 263 | var updateRate = 0 264 | 265 | val paramBlocks = model.symbol.listArguments() 266 | .filter(x => x != "data" && x != "softmax_label") 267 | .zipWithIndex.map { x => 268 | val state = opt.createState(x._2, model.argsDict(x._1)) 269 | (x._2, model.argsDict(x._1), model.gradDict(x._1), state, x._1) 270 | }.toArray 271 | 272 | for (iter <- 0 until epoch) { 273 | start = System.currentTimeMillis() 274 | numCorrect = 0f 275 | numTotal = 0f 276 | updateRate = 0 277 | 278 | for (begin <- 0 until trainBatches.length by batchSize) { 279 | val (batchD, batchL) = { 280 | if (begin + batchSize <= trainBatches.length) { 281 | val datas = trainBatches.drop(begin).take(batchSize) 282 | val labels = trainLabels.drop(begin).take(batchSize) 283 | (datas, labels) 284 | } else { 285 | val right = (begin + batchSize) - trainBatches.length 286 | val left = trainBatches.length - begin 287 | val datas = trainBatches.drop(begin).take(left) ++ trainBatches.take(right) 288 | val labels = trainLabels.drop(begin).take(left) ++ trainLabels.take(right) 289 | (datas, labels) 290 | } 291 | } 292 | numTotal += batchSize 293 | model.data.set(batchD.flatten.flatten) 294 | model.label.set(batchL) 295 | 296 | model.cnnExec.forward(isTrain = true) 297 | model.cnnExec.backward() 298 | 299 | val tmpCorrect = { 300 | val predLabel = NDArray.api.argmax_channel(model.cnnExec.outputs(0)) 301 | val result = predLabel.toArray.zip(batchL).map { predLabel => 302 | if (predLabel._1 == predLabel._2) 1 303 | else 0 304 | }.sum.toFloat 305 | predLabel.dispose() 306 | result 307 | } 308 | 309 | numCorrect = numCorrect + tmpCorrect 310 | val norm = Math.sqrt(paramBlocks.map { case (idx, weight, grad, state, name) => 311 | val tmp = grad / batchSize 312 | val l2Norm = NDArray.api.norm(tmp) 313 | val result = l2Norm.toScalar * l2Norm.toScalar 314 | l2Norm.dispose() 315 | tmp.dispose() 316 | result 317 | }.sum).toFloat 318 | 319 | paramBlocks.foreach { case (idx, weight, grad, state, name) => 320 | if (norm > maxGradNorm) { 321 | grad.set(grad.toArray.map(_ * (maxGradNorm / norm))) 322 | opt.update(idx, weight, grad, state) 323 | } 324 | else opt.update(idx, weight, grad, state) 325 | } 326 | } 327 | 328 | // end of training loop 329 | end = System.currentTimeMillis() 330 | println(s"Epoch $iter Training Time: ${(end - start) / 1000}," + 331 | s"Training Accuracy: ${numCorrect / numTotal * 100}%") 332 | 333 | // eval on dev set 334 | numCorrect = 0f 335 | numTotal = 0f 336 | for (begin <- 0 until devBatches.length by batchSize) { 337 | // println(s"dev $begin") 338 | if (begin + batchSize <= devBatches.length) { 339 | numTotal += batchSize 340 | val (batchD, batchL) = { 341 | val datas = devBatches.drop(begin).take(batchSize) 342 | val labels = devLabels.drop(begin).take(batchSize) 343 | (datas, labels) 344 | } 345 | 346 | model.data.set(batchD.flatten.flatten) 347 | model.label.set(batchL) 348 | 349 | model.cnnExec.forward(isTrain = false) 350 | 351 | val tmpCorrect = { 352 | val predLabel = NDArray.api.argmax_channel(model.cnnExec.outputs(0)) 353 | val result = predLabel.toArray.zip(batchL).map { predLabel => 354 | if (predLabel._1 == predLabel._2) 1 355 | else 0 356 | }.sum.toFloat 357 | predLabel.dispose() 358 | result 359 | } 360 | numCorrect = numCorrect + tmpCorrect 361 | } 362 | } 363 | val tmpAcc = numCorrect / numTotal 364 | println(s"Epoch $iter Test Accuracy: ${tmpAcc * 100}%") 365 | if (tmpAcc > maxAccuracy) { 366 | maxAccuracy = tmpAcc 367 | Model.saveCheckpoint(s"$saveModelPath/cnn-text-dev-acc-$maxAccuracy", 368 | iter, model.symbol, model.cnnExec.argDict, model.cnnExec.auxDict) 369 | println(s"Max accuracy on test set so far: ${maxAccuracy * 100}%") 370 | } 371 | 372 | // decay learning rate 373 | if (iter % 50 == 0 && iter > 0) { 374 | factor *= 0.5f 375 | opt.setLrMult(paramBlocks.map { x => Left(x._1) -> factor }.toMap) 376 | println(s"reset learning to ${opt.learningRate * factor}") 377 | } 378 | } 379 | } 380 | 381 | // MR 10-fold 382 | def main(args: Array[String]): Unit = { 383 | val exon = new AC_BLSTM_TextClassification 384 | val parser: CmdLineParser = new CmdLineParser(exon) 385 | try { 386 | parser.parseArgument(args.toList.asJava) 387 | println("Loading data...") 388 | 389 | var (numEmbed, word2vec) = 390 | if (exon.w2vFormatBin == 1) DataHelper.loadGoogleModel(exon.w2vFilePath) 391 | else DataHelper.loadPretrainedWord2vec(exon.w2vFilePath) 392 | 393 | val (datas, labels) = DataHelper.loadMSDataWithWord2vec( 394 | exon.mrDatasetPath, numEmbed, word2vec) 395 | 396 | val inputChannel = 1 397 | 398 | val randIdx = Source.fromFile(s"${exon.mrDatasetPath}/list.txt").mkString.split("\n").map(_.toInt) 399 | var crossIdx = exon.crossId 400 | 401 | // split train/dev set 402 | val (trainDats, devDatas) = { 403 | val train = { 404 | val l = randIdx.take(crossIdx * 1000).map(datas(_)).toArray 405 | val r = randIdx.drop((crossIdx + 1) * 1000).map(datas(_)).toArray 406 | l ++ r 407 | } 408 | val dev = randIdx.drop(crossIdx * 1000).take(1000).map(datas(_)).toArray 409 | (train, dev) 410 | } 411 | val (trainLabels, devLabels) = { 412 | val train = { 413 | val l = randIdx.take(crossIdx * 1000).map(labels(_)).toArray 414 | val r = randIdx.drop((crossIdx + 1) * 1000).map(labels(_)).toArray 415 | l ++ r 416 | } 417 | val dev = randIdx.drop(crossIdx * 1000).take(1000).map(labels(_)).toArray 418 | (train, dev) 419 | } 420 | 421 | val sentenceSize = datas(0).length / inputChannel 422 | println(sentenceSize) 423 | 424 | val ctx = if (exon.gpu == -1) Context.cpu() else Context.gpu(exon.gpu) 425 | 426 | val cnnModel = setupCnnModel(ctx, exon.batchSize, sentenceSize, numEmbed, 427 | inputChannels = inputChannel, 428 | numLstmLayer = 4, 429 | numHidden = 100, 430 | numFilter = 100, 431 | filterList = Array(2, 3, 4), 432 | initializer = new Xavier(factorType = "in", magnitude = 2.34f), 433 | resumeModelPath = exon.resumeModelPath) 434 | 435 | trainCNN(cnnModel, trainDats, trainLabels, devDatas, devLabels, exon.batchSize, 436 | exon.saveModelPath, learningRate = exon.lr) 437 | 438 | } catch { 439 | case ex: Exception => { 440 | logger.error(ex.getMessage, ex) 441 | parser.printUsage(System.err) 442 | sys.exit(1) 443 | } 444 | } 445 | } 446 | } 447 | 448 | class AC_BLSTM_TextClassification { 449 | @Option(name = "--cross-validation-id", usage = "the cross validation test set id, 0 ~ 9") 450 | private var crossId: Int = 0 451 | @Option(name = "--lr", usage = "the initial learning rate") 452 | private var lr: Float = 0.001f 453 | @Option(name = "--batch-size", usage = "the batch size") 454 | private var batchSize: Int = 100 455 | @Option(name = "--gpu", usage = "which gpu card to use, default is -1, means using cpu") 456 | private var gpu: Int = -1 457 | @Option(name = "--w2v-format-bin", usage = "does the word2vec file format is binary") 458 | private var w2vFormatBin: Int = 0 459 | @Option(name = "--mr-dataset-path", usage = "the MR polarity dataset path") 460 | private var mrDatasetPath: String = "" 461 | @Option(name = "--w2v-file-path", usage = "the word2vec file path") 462 | private var w2vFilePath: String = "" 463 | @Option(name = "--save-model-path", usage = "the model saving path") 464 | private var saveModelPath: String = "" 465 | @Option(name = "--resume-model-path", usage = "the model to be resumed") 466 | private var resumeModelPath: String = null 467 | } 468 | -------------------------------------------------------------------------------- /src/main/scala/experiments/DataHelper.scala: -------------------------------------------------------------------------------- 1 | package experiments 2 | 3 | import scala.io.Source 4 | import java.io.BufferedInputStream 5 | import java.io.FileInputStream 6 | import java.io.DataInputStream 7 | import java.io.InputStream 8 | import org.apache.mxnet.Random 9 | import org.apache.mxnet.Context 10 | import org.apache.mxnet.Shape 11 | 12 | /** 13 | * @author Depeng Liang 14 | */ 15 | object DataHelper { 16 | 17 | def cleanStr(str: String): String = { 18 | str.replaceAll("[^A-Za-z0-9(),!?'`]", " ") 19 | .replaceAll("'s", " 's") 20 | .replaceAll("'ve", " 've") 21 | .replaceAll("n't", " n't") 22 | .replaceAll("'re", " 're") 23 | .replaceAll("'d", " 'd") 24 | .replaceAll("'ll", " 'll") 25 | .replaceAll(",", " , ") 26 | .replaceAll("!", " ! ") 27 | .replaceAll("\\(", " \\( ") 28 | .replaceAll("\\)", " \\) ") 29 | .replaceAll("\\?", " \\? ") 30 | .replaceAll(" {2,}", " ") 31 | .trim() 32 | } 33 | 34 | // Loads MR polarity data from files, splits the data into words and generates labels. 35 | // Returns split sentences and labels. 36 | def loadMRDataAndLabels(dataPath: String): (Array[Array[String]], Array[Float]) = { 37 | // load data from file 38 | val positiveExamples = { 39 | val lines = Source.fromFile(s"$dataPath/rt-polarity.pos").mkString.split("\n") 40 | lines.map(_.trim()) 41 | } 42 | val negativeExamples = { 43 | val lines = Source.fromFile(s"$dataPath/rt-polarity.neg").mkString.split("\n") 44 | lines.map(_.trim()) 45 | } 46 | // split by words 47 | val xText = { 48 | val tmp = positiveExamples ++ negativeExamples 49 | tmp.map(cleanStr(_)).map(_.split(" ")) 50 | } 51 | // generate labels 52 | val positiveLabels = (1 to positiveExamples.length).map(x => 1).toArray 53 | val negativeLabels = (1 to negativeExamples.length).map(x => 0).toArray 54 | val y = positiveLabels ++ negativeLabels 55 | (xText, y.map(_.toFloat)) 56 | } 57 | 58 | // Pads all sentences to the same length. The length is defined by the longest sentence. 59 | // Returns padded sentences. 60 | def padSentences(sentences: Array[Array[String]], 61 | paddingWord: String = ""): Array[Array[String]] = { 62 | val sequenceLength = (-1 /: sentences.map(_.length)){ (max, len) => 63 | if (max < len) len else max 64 | } 65 | val paddedSetences = sentences.map { sentence => 66 | val numPadding = sequenceLength - sentence.length 67 | sentence ++ (1 to numPadding).map(x => paddingWord) 68 | } 69 | paddedSetences 70 | } 71 | 72 | def loadPretrainedWord2vec(inFile: String): (Int, Map[String, Array[Float]]) = { 73 | val lines = Source.fromFile(inFile).mkString.mkString.split("\n") 74 | val (vocabSize, dim) = { 75 | val head = lines(0).split(" ").map(_.toInt) 76 | (head(0), head(1)) 77 | } 78 | val word2vec = lines.drop(1).map { line => 79 | val tks = line.trim().split(" ") 80 | tks(0) -> tks.drop(1).map(_.toFloat) 81 | }.toMap 82 | (dim, word2vec) 83 | } 84 | 85 | def readString(dis: DataInputStream): String = { 86 | val MAX_SIZE = 50 87 | var bytes = new Array[Byte](MAX_SIZE) 88 | var b = dis.readByte() 89 | var i = -1 90 | val sb = new StringBuilder() 91 | while (b != 32 && b != 10) { 92 | i = i + 1 93 | bytes(i) = b 94 | b = dis.readByte() 95 | if (i == 49) { 96 | sb.append(new String(bytes)) 97 | i = -1 98 | bytes = new Array[Byte](MAX_SIZE) 99 | } 100 | } 101 | sb.append(new String(bytes, 0, i + 1)) 102 | sb.toString() 103 | } 104 | 105 | def getFloat(b: Array[Byte]): Float = { 106 | var accum = 0 107 | accum = accum | (b(0) & 0xff) << 0 108 | accum = accum | (b(1) & 0xff) << 8 109 | accum = accum | (b(2) & 0xff) << 16 110 | accum = accum | (b(3) & 0xff) << 24 111 | java.lang.Float.intBitsToFloat(accum).toFloat 112 | } 113 | 114 | def readFloat(is: InputStream): Float = { 115 | val bytes = new Array[Byte](4) 116 | is.read(bytes) 117 | getFloat(bytes) 118 | } 119 | 120 | // Reference https://github.com/NLPchina/Word2VEC_java 121 | def loadGoogleModel(path: String): (Int, Map[String, Array[Float]]) = { 122 | val bis = new BufferedInputStream(new FileInputStream(path)) 123 | val dis = new DataInputStream(bis) 124 | val wordSize = Integer.parseInt(readString(dis)) 125 | val dim = Integer.parseInt(readString(dis)) 126 | var word2vec = Map[String, Array[Float]]() 127 | for (i <- 0 until wordSize) { 128 | val word = readString(dis) 129 | val vectors = (1 to dim).map(j => readFloat(dis)).toArray 130 | word2vec += word -> vectors 131 | } 132 | bis.close() 133 | dis.close() 134 | (dim, word2vec) 135 | } 136 | 137 | // Map sentences and labels to vectors based on a pretrained word2vec. 138 | def buildInputDataWithWord2vec(sentences: Array[Array[String]], embeddingSize: Int, 139 | word2vec: Map[String, Array[Float]]): Array[Array[Array[Float]]] = { 140 | val xVec = sentences.map { sentence => 141 | sentence.map { word => 142 | if (word2vec.contains(word)) word2vec(word) 143 | else Random.uniform(-0.25f, 0.25f, Shape(embeddingSize), Context.cpu()).toArray 144 | } 145 | } 146 | xVec 147 | } 148 | 149 | // MR 150 | def loadMSDataWithWord2vec(dataPath: String, embeddingSize: Int, 151 | word2vec: Map[String, Array[Float]]): (Array[Array[Array[Float]]], Array[Float]) = { 152 | // loads the MR dataset 153 | val (sentences, labels) = loadMRDataAndLabels(dataPath) 154 | val sentencesPadded = padSentences(sentences) 155 | (buildInputDataWithWord2vec(sentencesPadded, embeddingSize, word2vec), labels) 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/main/scala/experiments/G_AC_BLSTM.scala: -------------------------------------------------------------------------------- 1 | package experiments 2 | 3 | import org.kohsuke.args4j.{ CmdLineParser, Option } 4 | import org.slf4j.LoggerFactory 5 | import scala.collection.JavaConverters._ 6 | import org.apache.mxnet.Initializer 7 | import org.apache.mxnet.Uniform 8 | import org.apache.mxnet.Context 9 | import org.apache.mxnet.Symbol 10 | import org.apache.mxnet.Shape 11 | import org.apache.mxnet.NDArray 12 | import org.apache.mxnet.Executor 13 | import org.apache.mxnet.optimizer.RMSProp 14 | import org.apache.mxnet.Optimizer 15 | import org.apache.mxnet.Model 16 | import scala.util.Random 17 | import org.apache.mxnet.Xavier 18 | import org.apache.mxnet.optimizer.AdaDelta 19 | import org.apache.mxnet.optimizer.Adam 20 | import Ops._ 21 | import scala.io.Source 22 | import org.apache.mxnet.util.OptionConversion._ 23 | 24 | /** 25 | * Implementation of the paper 26 | * AC-BLSTM: Asymmetric Convolutional Bidirectional LSTM Networks for Text Classification 27 | * 28 | * @author Depeng Liang 29 | */ 30 | object G_AC_BLSTM { 31 | 32 | private val logger = LoggerFactory.getLogger(classOf[G_AC_BLSTM]) 33 | 34 | case class CNNModel(cnnExec: Executor, symbol: Symbol, data: NDArray, label: NDArray, 35 | dataGrad: NDArray, argsDict: Map[String, NDArray], gradDict: Map[String, NDArray]) 36 | 37 | case class CANModel(ganExec: Executor, symbol: Symbol, data: NDArray, 38 | argsDict: Map[String, NDArray], gradDict: Map[String, NDArray]) 39 | 40 | final case class LSTMState(c: Symbol, h: Symbol) 41 | final case class LSTMParam(i2hWeight: Symbol, i2hBias: Symbol, 42 | h2hWeight: Symbol, h2hBias: Symbol) 43 | 44 | // LSTM Cell symbol 45 | def lstm( 46 | numHidden: Int, 47 | inData: Symbol, 48 | prevState: LSTMState, 49 | param: LSTMParam, 50 | seqIdx: Int, 51 | layerIdx: Int, 52 | dropout: Float = 0f): LSTMState = { 53 | 54 | val inDataa = { 55 | if (dropout > 0f) Symbol.api.Dropout(inData, p = dropout) 56 | else inData 57 | } 58 | val i2h = Symbol.api.FullyConnected(inDataa, 59 | weight = param.i2hWeight, 60 | bias = param.i2hBias, 61 | num_hidden = numHidden * 4, 62 | name = s"t${seqIdx}_l${layerIdx}_i2h") 63 | val h2h = Symbol.api.FullyConnected(prevState.h, 64 | weight = param.h2hWeight, 65 | bias = param.h2hBias, 66 | num_hidden = numHidden * 4, 67 | name = s"t${seqIdx}_l${layerIdx}_h2h") 68 | val gates = i2h + h2h 69 | val sliceGates = Symbol.api.SliceChannel(gates, num_outputs = 4, name = s"t${seqIdx}_l${layerIdx}_slice") 70 | val ingate = Symbol.api.sigmoid(sliceGates.get(0)) 71 | val inTransform = Symbol.api.tanh(sliceGates.get(1)) 72 | val forgetGate = Symbol.api.sigmoid(sliceGates.get(2)) 73 | val outGate = Symbol.api.tanh(sliceGates.get(3)) 74 | val nextC = (forgetGate * prevState.c) + (ingate * inTransform) 75 | val nextH = outGate * Symbol.api.tanh(nextC) 76 | LSTMState(c = nextC, h = nextH) 77 | } 78 | 79 | def makeTextCNN(sentenceSize: Int, numEmbed: Int, batchSize: Int, numHidden: Int, numLstmLayer: Int = 1, 80 | numLabel: Int = 2, filterList: Array[Int] = Array(3, 4, 5), numFilter: Int = 100, dropout: Float = 0.5f): Symbol = { 81 | 82 | var inputX = Symbol.Variable("data") 83 | val inputY = Symbol.Variable("softmax_label") 84 | 85 | // add dropout before conv layers 86 | if (dropout > 0f) inputX = Symbol.api.Dropout(inputX, p = dropout) 87 | 88 | val newNumFilter = numFilter 89 | val totalDim = newNumFilter * filterList.length 90 | var seqLen = sentenceSize - filterList.sorted.reverse.head + 1 91 | 92 | val windowSeqOutputs = filterList.map { filterSize => 93 | // inception v4 2 94 | var conv = Symbol.api.Convolution(inputX, kernel = Shape(1, numEmbed), num_filter = numFilter, cudnn_off = true) 95 | var bn = Symbol.api.BatchNorm(conv) 96 | var relu = Symbol.api.relu(bn) 97 | 98 | val len = sentenceSize - filterSize + 1 99 | 100 | conv = Symbol.api.Convolution(relu, kernel = Shape(filterSize, 1), num_filter = numFilter, cudnn_off = true, dilate = Shape(1, 1)) 101 | bn = Symbol.api.BatchNorm(conv) 102 | relu = Symbol.api.relu(bn) 103 | 104 | if (len > seqLen) { 105 | val partOne = Symbol.api.slice_axis(relu, axis = 2, begin = 0, end = (seqLen - 1)) 106 | var partTwo = Symbol.api.slice_axis(relu, axis = 2, begin = (seqLen - 1), end = len) 107 | partTwo = Symbol.api.Flatten(partTwo) 108 | partTwo = Symbol.api.FullyConnected(partTwo, num_hidden = newNumFilter) 109 | partTwo = Symbol.api.Reshape(partTwo, target_shape = Shape(batchSize, numFilter, 1, 1)) 110 | Symbol.api.Concat(Array(partOne, partTwo), num_args = 2, dim = 2) 111 | } else relu 112 | } 113 | 114 | val lstmInputs = { 115 | val concats = Symbol.api.Concat(windowSeqOutputs, num_args = windowSeqOutputs.length, dim = 1) 116 | Symbol.api.SliceChannel(concats, axis = 2, num_outputs = seqLen, squeeze_axis = true) 117 | } 118 | 119 | // bi-lstm 120 | var forwardParamCells = Array[LSTMParam]() 121 | var forwardLastStates = Array[LSTMState]() 122 | for (i <- 0 until numLstmLayer) { 123 | forwardParamCells = forwardParamCells :+ LSTMParam(i2hWeight = Symbol.Variable(s"f_l${i}_i2h_weight"), 124 | i2hBias = Symbol.Variable(s"f_l${i}_i2h_bias"), 125 | h2hWeight = Symbol.Variable(s"f_l${i}_h2h_weight"), 126 | h2hBias = Symbol.Variable(s"f_l${i}_h2h_bias")) 127 | forwardLastStates = forwardLastStates :+ LSTMState(c = Symbol.Variable(s"f_l${i}_init_c"), 128 | h = Symbol.Variable(s"f_l${i}_init_h")) 129 | } 130 | assert(forwardLastStates.length == numLstmLayer) 131 | 132 | var backwardParamCells = Array[LSTMParam]() 133 | var backwardLastStates = Array[LSTMState]() 134 | for (i <- 0 until numLstmLayer) { 135 | backwardParamCells = backwardParamCells :+ LSTMParam(i2hWeight = Symbol.Variable(s"b_l${i}_i2h_weight"), 136 | i2hBias = Symbol.Variable(s"b_l${i}_i2h_bias"), 137 | h2hWeight = Symbol.Variable(s"b_l${i}_h2h_weight"), 138 | h2hBias = Symbol.Variable(s"b_l${i}_h2h_bias")) 139 | backwardLastStates = backwardLastStates :+ LSTMState(c = Symbol.Variable(s"b_l${i}_init_c"), 140 | h = Symbol.Variable(s"b_l${i}_init_h")) 141 | } 142 | assert(backwardLastStates.length == numLstmLayer) 143 | 144 | // forward 145 | var forwardHiddenAll = Array[Symbol]() 146 | var dpRatio = 0f 147 | var hidden: Symbol = null 148 | for (seqIdx <- 0 until seqLen) { 149 | hidden = lstmInputs.get(seqIdx) 150 | // stack LSTM 151 | for (i <- 0 until numLstmLayer) { 152 | if (i == 0) dpRatio = 0f else dpRatio = dropout 153 | val nextState = lstm(numHidden, inData = hidden, 154 | prevState = forwardLastStates(i), 155 | param = forwardParamCells(i), 156 | seqIdx = seqIdx, layerIdx = i, dropout = dpRatio) 157 | hidden = nextState.h 158 | forwardLastStates(i) = nextState 159 | } 160 | // add dropout before softmax 161 | if (dropout > 0f) hidden = Symbol.api.Dropout(hidden, p = dropout) 162 | forwardHiddenAll = forwardHiddenAll :+ hidden 163 | } 164 | 165 | // backward 166 | var badkwardHiddenAll = Array[Symbol]() 167 | dpRatio = 0f 168 | for (seqIdx <- 0 until seqLen) { 169 | val k = seqLen - seqIdx - 1 170 | hidden = lstmInputs.get(k) 171 | // stack LSTM 172 | for (i <- 0 until numLstmLayer) { 173 | if (i == 0) dpRatio = 0f else dpRatio = dropout 174 | val nextState = lstm(numHidden, inData = hidden, 175 | prevState = backwardLastStates(i), 176 | param = backwardParamCells(i), 177 | seqIdx = k, layerIdx = i, dropout = dpRatio) 178 | hidden = nextState.h 179 | backwardLastStates(i) = nextState 180 | } 181 | // add dropout before softmax 182 | if (dropout > 0f) hidden = Symbol.api.Dropout(hidden, p = dropout) 183 | badkwardHiddenAll = hidden +: badkwardHiddenAll 184 | } 185 | 186 | val syms = forwardHiddenAll.zip(badkwardHiddenAll).map { case (f, b) => 187 | var tmp = Symbol.api.Concat(Array(f, b), num_args = 2, dim = 1) 188 | Symbol.api.Dropout(tmp, p = 0.5f) 189 | } 190 | 191 | var hiddenConcat = Symbol.api.Concat(syms, num_args = syms.length, dim = 1) 192 | val fc = Symbol.api.FullyConnected(hiddenConcat, num_hidden = numLabel) 193 | val sm = Symbol.api.SoftmaxOutput(fc, label = inputY) 194 | sm 195 | } 196 | 197 | def dcGan(oShape: Shape, finalAct: String, ngf: Int = 25): Symbol = { 198 | val h = oShape(oShape.length - 2) / 4 199 | val w = oShape(oShape.length - 1) / 4 200 | 201 | val channel = 4 202 | 203 | val code = Symbol.Variable("code") 204 | 205 | var net = Symbol.api.FullyConnected(code, num_hidden = h * w * ngf * channel, no_bias = true, name = "g1") 206 | net = Symbol.api.relu(net, name = "gact1") 207 | // 4 x 4 208 | net = Symbol.api.Reshape(net, shape = Shape(-1, ngf * channel, h, w)) 209 | // 8 x 8 210 | net = deconv2DBnRelu(net, prefix = "g2", iShape = Shape(ngf * channel, h, w) , oShape = Shape(ngf * 2, h * 2, w * 2), kShape = (4, 4)) 211 | // 16x16 212 | net = deconv2DBnRelu(net, iShape = Shape(ngf * 2, h * 2, w * 2), oShape = Shape(ngf, h * 4, w * 4), kShape = (4, 4), prefix = "g3") 213 | // 32 x 32 214 | net = deconv2DAct(net, prefix = "g4", actType = finalAct, iShape = Shape(ngf, h * 4, w * 4), 215 | oShape = Shape(oShape.toArray.takeRight(3)), kShape = (4, 4)) 216 | net 217 | } 218 | 219 | def setupGanModel(ctx: Context, batchSize: Int, sentenceSize: Int, numEmbed: Int, 220 | inputChannels: Int = 1, numFilter: Int = 25, initializer: Initializer = new Uniform(0.1f), 221 | resumeGanModelPath: String = null): CANModel = { 222 | 223 | val oShape = Shape(inputChannels, sentenceSize, numEmbed) 224 | val gan = dcGan(oShape, "relu", ngf = numFilter) 225 | 226 | val argNames = gan.listArguments() 227 | val auxNames = gan.listAuxiliaryStates() 228 | 229 | val dataShapes = Map("code" -> Shape(batchSize, 100)) 230 | 231 | val (argShapes, outShapes, auxShapes) = gan.inferShape(dataShapes) 232 | 233 | val argsDict = argNames.zip(argShapes).map { 234 | case (name, shape) => 235 | val nda = NDArray.zeros(shape, ctx) 236 | if (!dataShapes.contains(name)) { 237 | initializer(name, nda) 238 | } 239 | name -> nda 240 | }.toMap 241 | 242 | val argsGradDict = argNames.zip(argShapes) 243 | .filter(x => x._1 != "code") 244 | .map { x => 245 | val nda = NDArray.zeros(x._2, ctx) 246 | x._1 -> nda 247 | }.toMap 248 | 249 | if (resumeGanModelPath != null) { 250 | val pretrained = NDArray.load2Map(resumeGanModelPath) 251 | argsDict.foreach { 252 | case (name, ndA) => 253 | if (name != "code") { 254 | val key = s"arg:$name" 255 | if (pretrained.contains(key)) ndA.set(pretrained(key)) 256 | else logger.info(s"Skip argument $name") 257 | } 258 | } 259 | } 260 | 261 | val auxDict = auxNames.zip(auxShapes.map(NDArray.zeros(_, ctx))).toMap 262 | val ganExec = gan.bind(ctx, argsDict, argsGradDict, "write", auxDict, null, null) 263 | 264 | val data = argsDict("code") 265 | CANModel(ganExec, gan, data, argsDict, argsGradDict) 266 | } 267 | 268 | def setupCnnModel(ctx: Context, batchSize: Int, sentenceSize: Int, numEmbed: Int, numHidden: Int = 100, 269 | numLstmLayer: Int = 1, inputChannels: Int = 1, numLabel: Int = 2, numFilter: Int = 100, filterList: Array[Int] = Array(2, 3, 4), 270 | dropout: Float = 0.5f, initializer: Initializer = new Uniform(0.1f), resumeModelPath: String = null): CNNModel = { 271 | 272 | val cnn = makeTextCNN(sentenceSize, numEmbed, batchSize, numHidden, numLstmLayer, numLabel, filterList, numFilter, dropout) 273 | val argNames = cnn.listArguments() 274 | val auxNames = cnn.listAuxiliaryStates() 275 | 276 | // try bi-lstm 277 | val forwardInitC = for (l <- 0 until numLstmLayer) yield (s"f_l${l}_init_c", (batchSize, numHidden)) 278 | val fordwardInitH = for (l <- 0 until numLstmLayer) yield (s"f_l${l}_init_h", (batchSize, numHidden)) 279 | val backwardInitC = for (l <- 0 until numLstmLayer) yield (s"b_l${l}_init_c", (batchSize, numHidden)) 280 | val backwardInitH = for (l <- 0 until numLstmLayer) yield (s"b_l${l}_init_h", (batchSize, numHidden)) 281 | val initStates = (forwardInitC ++ fordwardInitH ++ backwardInitC ++ backwardInitH).map(x => x._1 -> Shape(x._2._1, x._2._2)).toMap 282 | 283 | val dataShapes = Map("data" -> Shape(batchSize, inputChannels, sentenceSize, numEmbed)) ++ initStates 284 | val labelNames = Array("softmax_label", "dloss_label") 285 | 286 | val (argShapes, outShapes, auxShapes) = cnn.inferShape(dataShapes) 287 | 288 | val argsDict = argNames.zip(argShapes).map { 289 | case (name, shape) => 290 | val nda = NDArray.zeros(shape, ctx) 291 | if (!dataShapes.contains(name) && !labelNames.contains(name)) { 292 | initializer(name, nda) 293 | } 294 | name -> nda 295 | }.toMap 296 | 297 | val argsGradDict = argNames.zip(argShapes) 298 | .filter(x => !labelNames.contains(x._1)) 299 | .map { x => 300 | val nda = NDArray.zeros(x._2, ctx) 301 | x._1 -> nda 302 | }.toMap 303 | 304 | if (resumeModelPath != null) { 305 | val pretrained = NDArray.load2Map(resumeModelPath) 306 | argsDict.foreach { 307 | case (name, ndA) => 308 | if (name != "softmax_label" && !dataShapes.contains(name)) { 309 | val key = s"arg:$name" 310 | if (pretrained.contains(key)) ndA.set(pretrained(key)) 311 | else logger.info(s"Skip argument $name") 312 | } 313 | } 314 | } 315 | 316 | val auxDict = auxNames.zip(auxShapes.map(NDArray.zeros(_, ctx))).toMap 317 | val cnnExec = cnn.bind(ctx, argsDict, argsGradDict, "write", auxDict, null, null) 318 | 319 | val data = argsDict("data") 320 | val dataGrad = argsGradDict("data") 321 | val label = argsDict("softmax_label") 322 | CNNModel(cnnExec, cnn, data, label, dataGrad, argsDict, argsGradDict) 323 | } 324 | 325 | def trainCNN(model: CNNModel, ganModel: CANModel, ganBatch: Int, fakeLabel: Int, 326 | trainBatches: Array[Array[Array[Float]]], trainLabels: Array[Float], 327 | devBatches: Array[Array[Array[Float]]], devLabels: Array[Float], 328 | batchSize: Int, saveModelPath: String, learningRate: Float = 0.001f): Unit = { 329 | val maxGradNorm = 0.5f 330 | val epoch = 1000 331 | val opt = new RMSProp(learningRate = learningRate, wd = 0.001f) 332 | var start = 0L 333 | var end = 0L 334 | var numCorrect = 0f 335 | var numTotal = 0f 336 | var factor = 0.5f 337 | var maxAccuracy = -1f 338 | var updateRate = 0 339 | 340 | val paramBlocks = model.symbol.listArguments() 341 | .filter(x => x != "softmax_label" && x != "dloss_label") 342 | .zipWithIndex.map { x => 343 | val state = opt.createState(x._2, model.argsDict(x._1)) 344 | (x._2, model.argsDict(x._1), model.gradDict(x._1), state, x._1) 345 | }.toArray 346 | 347 | val ganParamBlocks = ganModel.symbol.listArguments() 348 | .filter(x => x != "code") 349 | .zipWithIndex.map { x => 350 | val state = opt.createState(x._2, ganModel.argsDict(x._1)) 351 | (x._2, ganModel.argsDict(x._1), ganModel.gradDict(x._1), state, x._1) 352 | }.toArray 353 | 354 | val realBatch = batchSize - ganBatch 355 | val fakeLabels = Array.fill[Float](ganBatch)(fakeLabel) 356 | val shape = model.data.shape 357 | val fakeeee = Array(ganBatch) ++ shape.toArray.takeRight(3) 358 | val fakeArr = NDArray.empty(model.data.context, fakeeee: _*) 359 | 360 | for (iter <- 0 until epoch) { 361 | start = System.currentTimeMillis() 362 | numCorrect = 0f 363 | numTotal = 0f 364 | updateRate = 0 365 | 366 | for (begin <- 0 until trainBatches.length by realBatch) { 367 | // draw samples from training set, batchSize - ganBatch 368 | val (batchD, batchL) = { 369 | if (begin + realBatch <= trainBatches.length) { 370 | val datas = trainBatches.drop(begin).take(realBatch) 371 | val labels = trainLabels.drop(begin).take(realBatch) 372 | (datas, labels) 373 | } else { 374 | val right = (begin + realBatch) - trainBatches.length 375 | val left = trainBatches.length - begin 376 | val datas = trainBatches.drop(begin).take(left) ++ trainBatches.take(right) 377 | val labels = trainLabels.drop(begin).take(left) ++ trainLabels.take(right) 378 | (datas, labels) 379 | } 380 | } 381 | 382 | // draw samples from Generator, ganBatch 383 | val randomInput = 384 | org.apache.mxnet.Random.normal(0, 1.0f, ganModel.data.shape, ganModel.data.context) 385 | 386 | ganModel.data.set(randomInput) 387 | ganModel.ganExec.forward(isTrain = true) 388 | 389 | val outG = ganModel.ganExec.outputs(0) 390 | 391 | numTotal += batchSize 392 | 393 | model.data.set(batchD.flatten.flatten ++ outG.toArray) 394 | model.label.set(batchL ++ fakeLabels) 395 | 396 | model.cnnExec.forward(isTrain = true) 397 | model.cnnExec.backward() 398 | 399 | randomInput.dispose() 400 | 401 | val inputGrad = model.dataGrad.toArray.takeRight(outG.shape.product) 402 | fakeArr.set(inputGrad) 403 | 404 | ganModel.ganExec.backward(fakeArr) 405 | 406 | val tmpCorrect = { 407 | val predLabel = NDArray.api.argmax_channel(model.cnnExec.outputs(0)) 408 | val result = predLabel.toArray.zip(batchL).map { predLabel => 409 | if (predLabel._1 == predLabel._2) 1 410 | else 0 411 | }.sum.toFloat 412 | predLabel.dispose() 413 | result 414 | } 415 | 416 | numCorrect = numCorrect + tmpCorrect 417 | val norm = Math.sqrt(paramBlocks.map { 418 | case (idx, weight, grad, state, name) => 419 | val tmp = grad / batchSize 420 | val l2Norm = NDArray.api.norm(tmp) 421 | val result = l2Norm.toScalar * l2Norm.toScalar 422 | l2Norm.dispose() 423 | tmp.dispose() 424 | result 425 | }.sum).toFloat 426 | 427 | paramBlocks.foreach { 428 | case (idx, weight, grad, state, name) => 429 | if (norm > maxGradNorm) { 430 | grad.set(grad.toArray.map(_ * (maxGradNorm / norm))) 431 | opt.update(idx, weight, grad, state) 432 | } else opt.update(idx, weight, grad, state) 433 | opt.update(idx, weight, grad, state) 434 | } 435 | 436 | ganParamBlocks.foreach { 437 | case (idx, weight, grad, state, name) => 438 | opt.update(idx, weight, grad, state) 439 | } 440 | } 441 | 442 | // end of training loop 443 | end = System.currentTimeMillis() 444 | println(s"Epoch $iter Training Time: ${(end - start) / 1000}," + 445 | s"Training Accuracy: ${numCorrect / numTotal * 100}%") 446 | 447 | // eval on dev set 448 | numCorrect = 0f 449 | numTotal = 0f 450 | for (begin <- 0 until devBatches.length by batchSize) { 451 | if (begin + batchSize <= devBatches.length) { 452 | numTotal += batchSize 453 | val (batchD, batchL) = { 454 | val datas = devBatches.drop(begin).take(batchSize) 455 | val labels = devLabels.drop(begin).take(batchSize) 456 | (datas, labels) 457 | } 458 | 459 | model.data.set(batchD.flatten.flatten) 460 | model.label.set(batchL) 461 | 462 | model.cnnExec.forward(isTrain = false) 463 | 464 | val tmpCorrect = { 465 | val predLabel = { 466 | val outs = model.cnnExec.outputs(0) 467 | val arr = outs.toArray.grouped(fakeLabel + 1).map { gs => 468 | val tmp = gs.dropRight(1) 469 | ((tmp(0), 0) /: tmp.zipWithIndex) { (max, elem) => 470 | if (elem._1 > max._1) elem else max 471 | }._2 472 | } 473 | arr.toArray.map(_.toFloat) 474 | } 475 | val result = predLabel.toArray.zip(batchL).map { predLabel => 476 | if (predLabel._1 == predLabel._2) 1 477 | else 0 478 | }.sum.toFloat 479 | result 480 | } 481 | numCorrect = numCorrect + tmpCorrect 482 | } 483 | } 484 | val tmpAcc = numCorrect / numTotal 485 | println(s"Epoch $iter Test Accuracy : ${tmpAcc * 100}%") 486 | if (tmpAcc > maxAccuracy) { 487 | maxAccuracy = tmpAcc 488 | Model.saveCheckpoint(s"$saveModelPath/cnn-text-dev-acc-$maxAccuracy", 489 | iter, model.symbol, model.cnnExec.argDict, model.cnnExec.auxDict) 490 | Model.saveCheckpoint(s"$saveModelPath/gan-text-dev-acc-$maxAccuracy", 491 | iter, ganModel.symbol, ganModel.ganExec.argDict, ganModel.ganExec.auxDict) 492 | println(s"Max accuracy on test set so far: ${maxAccuracy * 100}%") 493 | } 494 | 495 | // decay learning rate 496 | if (iter % 50 == 0 && iter > 0) { 497 | factor *= 0.5f 498 | opt.setLrMult(paramBlocks.map { x => Left(x._1) -> factor }.toMap) 499 | println(s"reset learning to ${opt.learningRate * factor}") 500 | } 501 | } 502 | } 503 | 504 | // MR 10-fold 505 | def main(args: Array[String]): Unit = { 506 | val antm = new G_AC_BLSTM 507 | val parser: CmdLineParser = new CmdLineParser(antm) 508 | try { 509 | parser.parseArgument(args.toList.asJava) 510 | 511 | println("Loading data...") 512 | var (numEmbed, word2vec) = 513 | if (antm.w2vFormatBin == 1) DataHelper.loadGoogleModel(antm.w2vFilePath) 514 | else DataHelper.loadPretrainedWord2vec(antm.w2vFilePath) 515 | 516 | val (datas, labels) = DataHelper.loadMSDataWithWord2vec( 517 | antm.mrDatasetPath, numEmbed, word2vec) 518 | 519 | val inputChannel = 1 520 | val randIdx = Source.fromFile(s"${antm.mrDatasetPath}/list.txt").mkString.split("\n").map(_.toInt) 521 | 522 | var crossIdx = antm.crossId 523 | 524 | // split train/dev set 525 | val (trainDats, devDatas) = { 526 | val train = { 527 | val l = randIdx.take(crossIdx * 1000).map(datas(_)).toArray 528 | val r = randIdx.drop((crossIdx + 1) * 1000).map(datas(_)).toArray 529 | l ++ r 530 | } 531 | val dev = randIdx.drop(crossIdx * 1000).take(1000).map(datas(_)).toArray 532 | (train, dev) 533 | } 534 | val (trainLabels, devLabels) = { 535 | val train = { 536 | val l = randIdx.take(crossIdx * 1000).map(labels(_)).toArray 537 | val r = randIdx.drop((crossIdx + 1) * 1000).map(labels(_)).toArray 538 | l ++ r 539 | } 540 | val dev = randIdx.drop(crossIdx * 1000).take(1000).map(labels(_)).toArray 541 | (train, dev) 542 | } 543 | 544 | // reshpae for convolution input 545 | val sentenceSize = datas(0).length / inputChannel 546 | 547 | println(s"sensize: $sentenceSize") 548 | 549 | val ctx = if (antm.gpu == -1) Context.cpu() else Context.gpu(antm.gpu) 550 | 551 | val numLabels = 2 552 | val cnnModel = setupCnnModel(ctx, antm.batchSize, sentenceSize, numEmbed, 553 | inputChannels = inputChannel, 554 | numLstmLayer = 4, 555 | numHidden = 100, 556 | numFilter = 100, 557 | filterList = Array(2, 3, 4), 558 | numLabel = numLabels + 1, 559 | initializer = new Xavier(factorType = "in", magnitude = 2.34f), 560 | resumeModelPath = antm.resumeModelPath) 561 | 562 | val ganBatch = antm.ganBatch 563 | val ganModel = setupGanModel(ctx, ganBatch, sentenceSize, numEmbed, 564 | inputChannel, 25, resumeGanModelPath = antm.resumeGanModelPath) 565 | 566 | trainCNN(cnnModel, ganModel, ganBatch, numLabels, trainDats, trainLabels, devDatas, 567 | devLabels, antm.batchSize, antm.saveModelPath, learningRate = antm.lr) 568 | 569 | } catch { 570 | case ex: Exception => { 571 | println(ex.getMessage, ex) 572 | parser.printUsage(System.err) 573 | sys.exit(1) 574 | } 575 | } 576 | } 577 | } 578 | 579 | class G_AC_BLSTM { 580 | @Option(name = "--cross-validation-id", usage = "the cross validation test set id, 0 ~ 9") 581 | private var crossId: Int = 0 582 | @Option(name = "--lr", usage = "the initial learning rate") 583 | private var lr: Float = 0.001f 584 | @Option(name = "--batch-size", usage = "the batch size") 585 | private var batchSize: Int = 100 586 | @Option(name = "--gan-batch", usage = "the batch size") 587 | private var ganBatch: Int = 100 588 | @Option(name = "--gpu", usage = "which gpu card to use, default is -1, means using cpu") 589 | private var gpu: Int = -1 590 | @Option(name = "--w2v-format-bin", usage = "does the word2vec file format is binary") 591 | private var w2vFormatBin: Int = 0 592 | @Option(name = "--mr-dataset-path", usage = "the MR polarity dataset path") 593 | private var mrDatasetPath: String = "" 594 | @Option(name = "--w2v-file-path", usage = "the word2vec file path") 595 | private var w2vFilePath: String = "" 596 | @Option(name = "--save-model-path", usage = "the model saving path") 597 | private var saveModelPath: String = "" 598 | @Option(name = "--resume-model-path", usage = "the model to be resumed") 599 | private var resumeModelPath: String = null 600 | @Option(name = "--resume-gan-model-path", usage = "the model to be resumed") 601 | private var resumeGanModelPath: String = null 602 | } 603 | -------------------------------------------------------------------------------- /src/main/scala/experiments/Ops.scala: -------------------------------------------------------------------------------- 1 | package experiments 2 | 3 | import org.apache.mxnet._ 4 | import org.apache.mxnet.util.OptionConversion._ 5 | 6 | object Ops { 7 | 8 | val eps: Float = 1e-5f + 1e-12f 9 | 10 | // a deconv layer that enlarges the feature map 11 | def deconv2D(data: Symbol, iShape: Shape, oShape: Shape, 12 | kShape: (Int, Int), name: String, stride: (Int, Int) = (2, 2)): Symbol = { 13 | val targetShape = Shape(oShape(oShape.length - 2), oShape(oShape.length - 1)) 14 | val net = Symbol.api.Deconvolution(data, 15 | kernel = Shape(kShape._1, kShape._2), 16 | stride = Shape(stride._1, stride._2), 17 | target_shape = targetShape, 18 | num_filter = oShape(0), 19 | no_bias = true, 20 | name = name) 21 | net 22 | } 23 | 24 | def deconv2DBnRelu(data: Symbol, prefix: String, 25 | iShape: Shape, oShape: Shape, kShape: (Int, Int)): Symbol = { 26 | var net = deconv2D(data, iShape, oShape, kShape, name = s"${prefix}_deconv") 27 | net = Symbol.api.BatchNorm(net, fix_gamma = true, name = s"${prefix}_bn", eps = eps) 28 | net = Symbol.api.Activation(net, act_type = "relu", name = s"${prefix}_act") 29 | net 30 | } 31 | 32 | def deconv2DAct(data: Symbol, prefix: String, actType: String, 33 | iShape: Shape, oShape: Shape, kShape: (Int, Int)): Symbol = { 34 | var net = deconv2D(data, iShape, oShape, kShape, name = s"${prefix}_deconv") 35 | net = Symbol.api.Activation(net, act_type = "relu", name = s"${prefix}_act") 36 | net 37 | } 38 | 39 | } 40 | --------------------------------------------------------------------------------