├── LICENSE ├── README-eng.md ├── README.md ├── crawl_data.py ├── dataset.py ├── evaluate.py ├── files ├── batang.ttf ├── cjk.json ├── credentials.json ├── hangeul_image.png ├── requirements.txt └── ttf_links.txt ├── load_data.py ├── logs └── debug.txt ├── model.py ├── pretrained ├── plain_melnyk │ ├── saved_model.pb │ └── variables │ │ ├── variables.data-00000-of-00001 │ │ └── variables.index └── plain_melnyk_evaluation │ ├── CAM_handwritten.png │ ├── CAM_printed.png │ ├── handwritten_CHOSUNG.png │ ├── handwritten_JONGSUNG.png │ ├── handwritten_JUNGSUNG.png │ ├── printed_CHOSUNG.png │ ├── printed_JONGSUNG.png │ └── printed_JUNGSUNG.png ├── train.py └── utils ├── CustomLayers.py ├── MelnykNet.py ├── korean_manager.py ├── model_architectures.py ├── model_components.py └── predict_char.py /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README-eng.md: -------------------------------------------------------------------------------- 1 | README.md 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # KoOCR-tensorflow (Korean README) 3 | 4 | Tensorflow 딥러닝 기반의 오픈 소스 한글 OCR 엔진. 5 | Open-source Korean OCR engine based on Tensorflow, deep-learning. 6 | 7 | ![](files/hangeul_image.png) 8 | 9 | ## 개요 10 | - 중국어, 일본어 등 유사한 언어의 뛰어난 OCR 인식 성능에 비해 한글 인식에 대해서는 활발한 연구가 이루어지 않았다. 11 | - 쉽게 사용 가능한 고성능의 한글 OCR 프로젝트, 라이브러리가 많지 않았다. 12 | - 중국어 인식(HCCR)등에 사용된 학습 방법, Model Architecture를 한글 인식에 적용하고 성능을 비교하였다. 13 | - 한글의 특수한 구조에 기인해 초성, 중성, 종성을 각각 따로 예측하는 Multi-output 모델을 구성했다. 14 | 15 | 16 | ## Method and Plans 17 | 18 | 19 | - [x] DirectMap: Online and Offline Handwritten Chinese Character Recognition: A Comprehensive Study and New Benchmark 20 | - [x] Fire-module based model, GWAP: Building Efficient CNN Architecture for Offline Handwritten Chinese Character Recognition 21 | - [x] High-performance network architecture, CAM, GAP/GWAP/GWOAP: A High-Performance CNN Method for Offline Handwritten Chinese Character Recognition and Visualization 22 | - [ ] Hybrid learning loss: Improving Discrimination Ability of Convolutional Neural Networks by Hybrid Learning 23 | - [ ] Adaptive Drop Weight, GSLRE: Building Fast and Compact Convolutional Neural Networks for Offline Handwritten Chinese Character Recognition 24 | - [x] Adversarial Feature Learning: Robust offline handwritten character recognition through exploring writer-independent features under the guidance of printed data 25 | - [ ] DenseRAN style model: DenseRAN for Offline Handwritten Chinese Character Recognition 26 | - [x] Iterative Refinemet: Improving Offline Handwritten Chinese Character Recognition by Iterative Refinement 27 | 28 | 위 논문들에서 제시하는 method를 구현하고 한글 인식에 대한 성능 개선의 정도를 평가하는 논문을 작성할 계획이다. 29 | 30 | ## 모델 성능 31 | 32 | pretrained weights는 `pretrained/해당모델` 폴더 아래 저장되어 있다. 각 모델에 대한 추가적인 학습결과는 `pretrained/해당모델_evaluation`에 저장되어 있다. Class별 confusion matrix, Grad-CAM 결과 등이 저장되어 있다. 아래 표는 각 모델과 학습 방법의 비교 결과이다. 33 | 34 | model type | 인쇄체 정확도 | 손글씨 정확도 | 추론 시간(ms/image) | 모델 크기(mb) 35 | ----------------------- | ------------ | ------------- | ------------------- | -------------- 36 | plain_melnyk_complete |99.94% |97.94% | |57.1 37 | 38 | ### plain_melnyk_complete 39 | High-performance network architecture(melnyk network) 의 baseline 모델. `0.001`의 학습률로 1 에포크, `0.00003`의 학습률로 2 에포크 학습한 결과이다. [Adabound optimizer](https://arxiv.org/abs/1902.09843)를 사용하였다. `fc_link`는 합성곱 신경망의 output feature map을 fully connected layer로 연결하는 방법을 의미하는데, 본 논문에서 제시한 GAP(Gradient Average Pooling)을 개선한 GWAP를 사용하였다. 40 | ``` 41 | !python train.py --learning_rate=0.00003 --optimizer=adabound --image_size=96 --weights=./logs/weights/ --fc_link=GWAP --batch_size=128 --epochs=2 --patch_size=20 42 | ``` 43 | 44 | 45 | ## 프로젝트 사용 46 | 47 | ### load_data.py 48 | ``` 49 | !python load_data.py --sevenzip=true 50 | ``` 51 | Google Drive에 업로드된 데이터셋을 다운로드 받아 `./data`, `./val_data` 에 저장한다. 데이터셋은 300여개의 .pickle 패치로 이루어져 있고, 데이터의 특성에 따라 `handwritten_*.pickle`, `printed_*.pickle`, `clova_*.pickle` 으로 이름지어져있다. 각각 손글씨, 인쇄체, 손글씨 폰트의 이미지를 나타낸다. 52 | 53 | `sevenzip` 변수는 압축된 데이터를 .7z 파일로 받을지 .zip 파일로 받을지를 나타낸다. True 값이 다운로드와 압축 속도가 빠르다. 54 | 55 | ### crawl_data.py 56 | 57 | ``` 58 | !python crawl_data.py --AIHub=true 59 | --clova=true 60 | --image_size=96 61 | --x_offset=8 62 | --y_offset=8 63 | --char_size=80 64 | ``` 65 | 데이터셋을 크롤링해서 다운로드받는다. load_data.py와 같은 역할을 한다. `x_offset`, `y_offset`, `char_size` 변수는 폰트를 이미지에 그릴 때 위치의 offset과 문자의 크기를 지정한다. 아래 표는 실험에서 사용한 이미지 크기에 따른 변수 설정값이다. 66 | 67 | image size | x_offset | y_offset | char_size 68 | ---------- | -------- | -------- | --------- 69 | 64 | 5| 5|50 70 | 96 |8 |8 |80 71 | 128 |14 |10 |100 72 | 256 |50 |10 |200 73 | 74 | 75 | ### model. py 76 | ``` 77 | import model 78 | OCR_model=model.KoOCR(weights='C:\\...', split_components=True, ...) 79 | 80 | OCR_model.model.summary() 81 | OCR_model.train(epochs=10, lr=0.01, ...) 82 | 83 | pred=OCR_model.predict(image, n=5) 84 | ``` 85 | 모델을 정의하는 모듈으로 `KoOCR` class가 정의되어 있다. **추론에 사용되는 메소드 `KoOCR.predict`는 이미지 혹은 이미지의 배치를 입력받아 가능성이 가장 높은 top-n 개의 한글 글자를 반환한다.** 모델의 추가적인 학습에 사용되는 메소드는 `KoOCR.train`으로, 입력받은 Hyperparameter를 바탕으로 학습을 진행한다. 86 | 87 | ### train. py 88 | ``` 89 | !python train.py--split_components=true 90 | --network=melnyk 91 | --image_size=96 92 | --direct_map=true 93 | --epochs=10 94 | ... 95 | ``` 96 | 학습을 진행하는 파이썬 모듈인지만, 모델을 정의하고 `KoOCR.train`을 호출하는 역할을 할 뿐, 직접 `model.py`를 import 하고 훈련하는 것과 차이가 없다. 학습결과와 과정에 대한 모든 정보는 `./logs`에 저장되고, 가중치는 매 에포크마다 `./logs/weights.h5`에 저장된다. 97 | 98 | ### evaluate. py 99 | ``` 100 | python evaluate.py --weights='./logs/weights.h5' 101 | --accuracy=true 102 | --confusion_matrix=true 103 | --class_activation=true 104 | ``` 105 | 모델을 정확도, confusion matrix, CAM 3가지 방법으로 분석한다. 각 방법을 선택 해제하거나 top-n 정확도 등 각 방법의 세부적인 parameter 또한 설정할 수 있다. 106 | -------------------------------------------------------------------------------- /crawl_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import urllib.request 3 | import argparse 4 | import numpy as np 5 | from PIL import Image 6 | from PIL import ImageDraw 7 | from PIL import ImageFont 8 | import random 9 | import PIL 10 | import json 11 | import _pickle as pickle #cPickle 12 | import progressbar 13 | import threading 14 | import collections 15 | import tensorflow as tf 16 | import utils.korean_manager as korean_manager 17 | from google_drive_downloader import GoogleDriveDownloader as gdd 18 | #bool type for arguments 19 | def str2bool(v): 20 | if isinstance(v, bool): 21 | return v 22 | if v.lower() in ('yes', 'true', 't', 'y', '1'): 23 | return True 24 | elif v.lower() in ('no', 'false', 'f', 'n', '0'): 25 | return False 26 | else: 27 | raise argparse.ArgumentTypeError('Boolean value expected.') 28 | #Define arguments 29 | parser = argparse.ArgumentParser(description='Download dataset') 30 | parser.add_argument("--font_path", type=str,default='./fonts') 31 | parser.add_argument("--AIHub_path", type=str,default='./AIhub') 32 | parser.add_argument("--pickle_path", type=str,default='./data') 33 | parser.add_argument("--val_path", type=str,default='./val_data') 34 | parser.add_argument("--pickle_path_val", type=str,default='./data') 35 | parser.add_argument("--val_ratio", type=float,default=.1) 36 | 37 | parser.add_argument("--clova", type=str2bool,default=False) 38 | parser.add_argument("--image_test", type=str2bool,default=False) 39 | parser.add_argument("--image_size", type=int,default=96) 40 | parser.add_argument("--x_offset", type=int,default=8) 41 | parser.add_argument("--y_offset", type=int,default=8) 42 | parser.add_argument("--char_size", type=int,default=80) 43 | 44 | parser.add_argument("--AIHub", type=str2bool,default=False) 45 | parser.add_argument("--pickle_size", type=int,default=5000) 46 | 47 | def crawl_dataset(): 48 | if args.clova: 49 | crawl_clova_fonts() 50 | charset=korean_manager.load_charset() 51 | convert_all_fonts(charset) 52 | if args.AIHub: 53 | download_AIHub_GoogleDrive() 54 | pickle_AIHub_images() 55 | 56 | def pickle_AIHub_images(): 57 | #Pickle AIHub data into handwritten, printed 58 | if os.path.isdir(args.pickle_path)==False: 59 | os.mkdir(args.pickle_path) 60 | if os.path.isdir(args.val_path)==False: 61 | os.mkdir(args.val_path) 62 | random.seed(42) 63 | #Unpickle Handwritten files 64 | f = open(os.path.join(args.AIHub_path,'handwritten_label.json')) 65 | anno = json.load(f) 66 | f.close() 67 | random.shuffle(anno['annotations']) 68 | 69 | images_before_pickle=args.pickle_size 70 | pickle_idx=0 71 | image_arr,label_arr=[],[] 72 | 73 | for x in progressbar.progressbar(anno['annotations']): 74 | #Save data split into pickle 75 | if images_before_pickle==0: 76 | images_before_pickle=args.pickle_size 77 | #Split train and test data 78 | if random.random()>args.val_ratio: 79 | file_path=args.pickle_path 80 | else: 81 | file_path=args.val_path 82 | 83 | with open(os.path.join(file_path,'handwritten_'+str(pickle_idx)+'.pickle'),'wb') as handle: 84 | pickle.dump({'image':np.array(image_arr),'label':np.array(label_arr)},handle) 85 | pickle_idx+=1 86 | image_arr,label_arr=[],[] 87 | #Append to list if character type of data 88 | if (x['attributes']['type']=='글자(음절)'): 89 | #Find the path between 2 directories 90 | true_path='' 91 | path1=os.path.join(args.AIHub_path,'1_syllable/'+x['image_id']+'.png') 92 | path2=os.path.join(args.AIHub_path,'2_syllable/'+x['image_id']+'.png') 93 | if os.path.isfile(path1)==True: 94 | true_path=path1 95 | elif os.path.isfile(path2)==True: 96 | true_path=path2 97 | #Save image and text 98 | if true_path: 99 | im=tf.keras.preprocessing.image.load_img(true_path,color_mode='grayscale',target_size=(args.image_size,args.image_size)) 100 | image_arr.append(tf.keras.preprocessing.image.img_to_array(im)[:,:,0]) 101 | label_arr.append(x['text']) 102 | 103 | os.remove(true_path) 104 | images_before_pickle-=1 105 | #Unpickle Printed files 106 | f = open(os.path.join(args.AIHub_path,'printed_label.json')) 107 | anno = json.load(f) 108 | f.close() 109 | random.shuffle(anno['annotations']) 110 | 111 | images_before_pickle=args.pickle_size 112 | pickle_idx=0 113 | image_arr,label_arr=[],[] 114 | 115 | for x in progressbar.progressbar(anno['annotations']): 116 | #Save data split into pickle 117 | if images_before_pickle==0: 118 | images_before_pickle=args.pickle_size 119 | #Split train and test data 120 | if random.random()>args.val_ratio: 121 | file_path=args.pickle_path 122 | else: 123 | file_path=args.val_path 124 | with open(os.path.join(file_path,'printed_'+str(pickle_idx)+'.pickle'),'wb') as handle: 125 | pickle.dump({'image':np.array(image_arr),'label':np.array(label_arr)},handle) 126 | pickle_idx+=1 127 | image_arr,label_arr=[],[] 128 | #Append to list if character type of data 129 | if (x['attributes']['type']=='글자(음절)'): 130 | path=os.path.join(args.AIHub_path,'syllable/'+x['image_id']+'.png') 131 | if os.path.isfile(path)==True: 132 | im=tf.keras.preprocessing.image.load_img(path,color_mode='grayscale',target_size=(args.image_size,args.image_size)) 133 | image_arr.append(tf.keras.preprocessing.image.img_to_array(im)[:,:,0]) 134 | label_arr.append(x['text']) 135 | 136 | os.remove(path) 137 | images_before_pickle-=1 138 | 139 | def download_AIHub_GoogleDrive(): 140 | #Download AIHUB OCR data from Google Drive 141 | handwritten_file_id_1='13GCWsztfD00mHxKGNVO_c6uxS_9J_JOY' 142 | handwritten_file_id_2='1N2dTwZ8TgYRFBeNDKgjxjDHqULk_JX6X' 143 | handwritten_label_id='1rX979OhUHCKSYRbBPaMIHtFQa0eVdSXt' 144 | printed_file_id_1='1MNYnv4aO0kWaDigb9iEcIdpxO_pF2s-m' 145 | printed_label_id='1ibZrGauMoM1E9Bx2fMGtiJEqQ6nh8Qy8' 146 | 147 | idlist_file=[[handwritten_file_id_1,'handwritten-1'],[handwritten_file_id_2,'handwritten-2'],[printed_file_id_1,'printed-1']] 148 | idlist_label=[[handwritten_label_id,'handwritten_label'],[printed_label_id,'printed_label']] 149 | 150 | if os.path.isdir(args.AIHub_path)==False: 151 | os.mkdir(args.AIHub_path) 152 | 153 | print("Downloading AIHUB OCR from Google Drive...") 154 | 155 | for file_id in idlist_file: 156 | zip_dest_path=os.path.join(args.AIHub_path,f'{file_id[1]}.zip') 157 | gdd.download_file_from_google_drive(file_id=file_id[0],dest_path=zip_dest_path,unzip=True) 158 | os.remove(zip_dest_path) 159 | for file_id in idlist_label: 160 | json_dest_path=os.path.join(args.AIHub_path,f'{file_id[1]}.json') 161 | gdd.download_file_from_google_drive(file_id=file_id[0],dest_path=json_dest_path) 162 | 163 | print("Download complete") 164 | 165 | def crawl_clova_fonts(): 166 | #Crawl and download .ttf files listed in ttf_links.txt 167 | #Make directory 168 | print('Downloading fonts in', args.font_path) 169 | download_path=args.font_path 170 | if os.path.isdir(download_path)==False: 171 | os.mkdir(download_path) 172 | 173 | #Retrieve all file locations 174 | url_path='files/ttf_links.txt' 175 | f = open(url_path, 'r') 176 | 177 | while True: 178 | url = f.readline() 179 | # if line is empty -> EOF 180 | if not url: 181 | break 182 | file_name=url.split('/')[-1].replace('\n','') 183 | url=url.replace('https://','') #Parse 'https://' 184 | url=urllib.parse.quote(url.replace('\n','')) #Change encoding 185 | 186 | urllib.request.urlretrieve('https://'+url, os.path.join(download_path,file_name)) 187 | f.close() 188 | 189 | def draw_single_char(ch, font): 190 | img = Image.new("L", (args.image_size, args.image_size), 255) 191 | draw = ImageDraw.Draw(img) 192 | draw.text((args.x_offset, args.y_offset), ch, 0, font=font) 193 | return img 194 | 195 | def font2img(font_path,font_idx,charset,save_dir): 196 | font=ImageFont.truetype(font_path,size=args.char_size) 197 | image_arr,label_arr=[],[] 198 | 199 | try: 200 | for c in charset: 201 | e = draw_single_char(c, font) 202 | image_arr.append(np.array(e)) 203 | label_arr.append(c) 204 | 205 | with open(os.path.join(save_dir,'clova_'+str(font_idx)+'.pickle'),'wb') as handle: 206 | pickle.dump({'image':np.array(image_arr),'label':np.array(label_arr)},handle) 207 | except: 208 | pass 209 | 210 | def convert_all_fonts(charset): 211 | font_directory=args.font_path 212 | save_directory=args.pickle_path 213 | 214 | if os.path.isdir(save_directory)==False: 215 | os.mkdir(save_directory) 216 | if os.path.isdir(args.val_path)==False: 217 | os.mkdir(args.val_path) 218 | 219 | fonts=os.listdir(font_directory) 220 | 221 | for idx,font in progressbar.progressbar(enumerate(fonts)): 222 | full_path=os.path.join(font_directory,font) 223 | 224 | font2img(full_path,idx,charset,save_directory) 225 | 226 | if __name__=='__main__': 227 | args = parser.parse_args() 228 | if args.image_test==False: 229 | crawl_dataset() 230 | 231 | else: 232 | default_font=ImageFont.truetype('files/batang.ttf',size=args.char_size) 233 | arr=draw_single_char('가',default_font) 234 | arr=np.array(arr) 235 | result = Image.fromarray(arr.astype(np.uint8)) 236 | result.save('./logs/가.jpg') 237 | 238 | arr=draw_single_char('나',default_font) 239 | arr=np.array(arr) 240 | result = Image.fromarray(arr.astype(np.uint8)) 241 | result.save('./logs/나.jpg') -------------------------------------------------------------------------------- /dataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | import tensorflow as tf 4 | import fnmatch 5 | import _pickle as pickle #cPickle 6 | import utils.korean_manager as korean_manager 7 | import random 8 | import progressbar 9 | 10 | class DataPickleLoader(): 11 | #Load data patch by patch 12 | def __init__(self,patch_size=10,data_path='./data',val_data_path='./val_data',split_components=True,val_data=.1, 13 | return_image_type=False,silent_mode=False): 14 | self.data_path=data_path 15 | self.val_data_path=val_data_path 16 | self.patch_size=patch_size 17 | self.split_components=split_components 18 | self.return_image_type=return_image_type 19 | self.current_idx=0 20 | self.silent_mode=silent_mode 21 | 22 | file_list=fnmatch.filter(os.listdir(data_path), '*.pickle') 23 | val_file_list=fnmatch.filter(os.listdir(val_data_path), '*.pickle') 24 | 25 | self.file_list=file_list 26 | self.val_file_list=val_file_list 27 | np.random.shuffle(self.val_file_list) 28 | self.mix_indicies() 29 | 30 | def load_pickle(self,path): 31 | with open(path,'rb') as handle: 32 | data=pickle.load(handle) 33 | return data 34 | 35 | def get_val(self,prob=0.3): 36 | data=self.load_pickle(os.path.join(self.val_data_path,self.val_file_list[0])) 37 | images=data['image'] 38 | 39 | if self.split_components==True: 40 | cho,jung,jong=korean_manager.korean_split_numpy(data['label']) 41 | else: 42 | labels=korean_manager.korean_numpy(data['label']) 43 | types=np.repeat(int(self.val_file_list[0].split('_')[0]=='handwritten'),images.shape[0]) 44 | 45 | for pkl in self.val_file_list[1:int(len(self.val_file_list)*prob)]: 46 | data=self.load_pickle(os.path.join(self.val_data_path,pkl)) 47 | images=np.concatenate((images,data['image']),axis=0) 48 | 49 | if self.split_components==True: 50 | cho_,jung_,jong_=korean_manager.korean_split_numpy(data['label']) 51 | 52 | cho=np.concatenate((cho,cho_)) 53 | jung=np.concatenate((jung,jung_)) 54 | jong=np.concatenate((jong,jong_)) 55 | types=np.concatenate((types, np.repeat(int(pkl.split('_')[0]=='handwritten'),data['image'].shape[0]))) 56 | else: 57 | labels=np.concatenate((labels,korean_manager.korean_numpy(data['label']))) 58 | 59 | #Random shuffle data 60 | ind_list = list(range(types.shape[0])) 61 | random.shuffle(ind_list) 62 | 63 | if self.split_components==True: 64 | #One hot encode labels and return 65 | cho=tf.one_hot(cho,len(korean_manager.CHOSUNG_LIST)) 66 | jung=tf.one_hot(jung,len(korean_manager.JUNGSUNG_LIST)) 67 | jong=tf.one_hot(jong,len(korean_manager.JONGSUNG_LIST)) 68 | 69 | if self.return_image_type: 70 | return images,{'CHOSUNG':cho,'JUNGSUNG':jung,'JONGSUNG':jong,'DISC':types} 71 | else: 72 | return images,{'CHOSUNG':cho,'JUNGSUNG':jung,'JONGSUNG':jong} 73 | else: 74 | return images,labels[ind_list] 75 | 76 | def get(self): 77 | print(f"Loading dataset patch {self.current_idx//self.patch_size}/{len(self.file_list)//self.patch_size+1}...") 78 | #Check if end of list 79 | did_reset=False 80 | next_idx=self.current_idx+self.patch_size 81 | if next_idx>len(self.file_list): 82 | next_idx=len(self.file_list) 83 | did_reset=True 84 | 85 | data=self.load_pickle(os.path.join(self.data_path,self.file_list[self.current_idx])) 86 | images=data['image'] 87 | 88 | if self.split_components==True: 89 | cho,jung,jong=korean_manager.korean_split_numpy(data['label']) 90 | else: 91 | labels=korean_manager.korean_numpy(data['label']) 92 | types=np.repeat(int(self.file_list[self.current_idx].split('_')[0]=='handwritten'),images.shape[0]) 93 | 94 | path_slice=self.file_list[self.current_idx+1:next_idx] 95 | 96 | if self.silent_mode: 97 | prog=path_slice 98 | else: 99 | prog=progressbar.progressbar(path_slice) 100 | for pkl in prog: 101 | data=self.load_pickle(os.path.join(self.data_path,pkl)) 102 | images=np.concatenate((images,data['image']),axis=0) 103 | 104 | if self.split_components==True: 105 | cho_,jung_,jong_=korean_manager.korean_split_numpy(data['label']) 106 | 107 | cho=np.concatenate((cho,cho_)) 108 | jung=np.concatenate((jung,jung_)) 109 | jong=np.concatenate((jong,jong_)) 110 | types=np.concatenate((types, np.repeat(int(pkl.split('_')[0]=='handwritten'),data['image'].shape[0]))) 111 | else: 112 | labels=np.concatenate((labels,korean_manager.korean_numpy(data['label']))) 113 | 114 | 115 | #Reset if final chunk of image 116 | self.current_idx=next_idx 117 | if self.current_idx==len(self.file_list): 118 | self.mix_indicies() 119 | self.current_idx=0 120 | 121 | ind_list = list(range(types.shape[0])) 122 | random.shuffle(ind_list) 123 | if self.split_components==True: 124 | #One hot encode labels and return 125 | cho=tf.one_hot(cho,len(korean_manager.CHOSUNG_LIST)) 126 | jung=tf.one_hot(jung,len(korean_manager.JUNGSUNG_LIST)) 127 | jong=tf.one_hot(jong,len(korean_manager.JONGSUNG_LIST)) 128 | if self.return_image_type: 129 | return images,{'CHOSUNG':cho,'JUNGSUNG':jung,'JONGSUNG':jong,'DISC':types},did_reset 130 | else: 131 | return images,{'CHOSUNG':cho,'JUNGSUNG':jung,'JONGSUNG':jong},did_reset 132 | else: 133 | labels=tf.one_hot(labels,len(korean_manager.load_charset())) 134 | return images,labels,did_reset 135 | 136 | def mix_indicies(self): 137 | np.random.shuffle(self.file_list) -------------------------------------------------------------------------------- /evaluate.py: -------------------------------------------------------------------------------- 1 | import model 2 | import dataset 3 | import tensorflow as tf 4 | import numpy as np 5 | import argparse 6 | import matplotlib.pyplot as plt 7 | import os 8 | import fnmatch 9 | import utils.korean_manager as korean_manager 10 | import progressbar 11 | import _pickle as pickle #cPickle 12 | from utils.CustomLayers import MultiOutputGradCAM 13 | from utils.model_components import PreprocessingPipeline 14 | from sklearn.metrics import confusion_matrix 15 | import seaborn as sn 16 | import pandas as pd 17 | import matplotlib.font_manager as font_manager 18 | #bool type for arguments 19 | def str2bool(v): 20 | if isinstance(v, bool): 21 | return v 22 | if v.lower() in ('yes', 'true', 't', 'y', '1'): 23 | return True 24 | elif v.lower() in ('no', 'false', 'f', 'n', '0'): 25 | return False 26 | else: 27 | raise argparse.ArgumentTypeError('Boolean value expected.') 28 | #Define arguments 29 | parser=argparse.ArgumentParser(description='Download dataset') 30 | parser.add_argument("--data_path", type=str,default='./val_data') 31 | parser.add_argument("--train_data_path", type=str,default='./data') 32 | parser.add_argument("--image_size", type=int,default=256) 33 | parser.add_argument("--split_components", type=str2bool,default=True) 34 | parser.add_argument("--patch_size", type=int,default=10) 35 | 36 | parser.add_argument("--show_augmentation", type=str2bool,default=False) 37 | parser.add_argument("--accuracy", type=str2bool,default=False) 38 | parser.add_argument("--confusion_matrix", type=str2bool,default=False) 39 | parser.add_argument("--class_activation", type=str2bool,default=False) 40 | 41 | parser.add_argument("--class_activation_n", type=int,default=10) 42 | 43 | parser.add_argument("--weights", type=str,default='') 44 | parser.add_argument("--top_n", type=int,default=5) 45 | 46 | def generate_CAM(model,key_text): 47 | #Pick one pickle, and load 10 data from file 48 | file_list=fnmatch.filter(os.listdir(args.data_path), f'{key_text}*.pickle') 49 | np.random.shuffle(file_list) 50 | with open(os.path.join(args.data_path,file_list[0]),'rb') as handle: 51 | data=pickle.load(handle) 52 | indicies=np.random.choice(len(data['image']), args.class_activation_n, replace=False) 53 | 54 | fig = plt.figure(figsize=(args.class_activation_n*2,8)) 55 | for idx,data_idx in enumerate(indicies): 56 | #Get Class Activation Map 57 | cam=MultiOutputGradCAM(model.model,data['label'][data_idx]) 58 | heatmap_list=cam.compute_heatmap(data['image'][data_idx]) 59 | heatmap_list.append(None) 60 | 61 | for comp in range(4): 62 | plt.subplot(4,args.class_activation_n,idx+1+args.class_activation_n*comp) 63 | plt.imshow(data['image'][data_idx],cmap='gray') 64 | if comp!=3: 65 | plt.imshow(heatmap_list[comp],alpha=0.5,cmap='jet') 66 | plt.axis('off') 67 | plt.savefig(f'./logs/CAM_{key_text}.png') 68 | plt.clf() 69 | 70 | def plot_augmentation(): 71 | images_per_type=3 72 | augment_times=5 73 | key_texts=['clova','handwritten','printed'] 74 | width,height=images_per_type*3,augment_times+1 75 | #Define PreprocessingPipeline with data augmentation. 76 | aug_model=PreprocessingPipeline(False,True) 77 | 78 | fig = plt.figure(figsize=(width,height)) 79 | for key_text_idx in range(3): 80 | #Read data from randomly selected batch 81 | key_text=key_texts[key_text_idx] 82 | file_list=fnmatch.filter(os.listdir(args.train_data_path), f'{key_text}*.pickle') 83 | np.random.shuffle(file_list) 84 | with open(os.path.join(args.train_data_path,file_list[0]),'rb') as handle: 85 | data=pickle.load(handle) 86 | #Plot first 3 images and augment_times augmented versions. 87 | for idx in range(images_per_type): 88 | plt.subplot(key_text_idx*3+idx+1,width,height) 89 | plt.imshow(data['image'][idx]) 90 | plt.axis('off') 91 | for k in range(1,augment_times+1): 92 | plt.subplot(key_text_idx*3+idx+1 + width*k,width,height) 93 | new_image=aug_model(data['image'][idx].reshape(1,96,96,1)) 94 | plt.imshow(new_image[0]) 95 | plt.axis('off') 96 | plt.savefig('./logs/Augmentation_Sample.png') 97 | plt.clf() 98 | 99 | def generate_confusion_matrix(model,key_text): 100 | #Generate confusion matrix of each component based on sklearn, 101 | #Only call when split_components is True. 102 | if not args.split_components: 103 | return 104 | try: 105 | path = './files/batang.ttf' 106 | prop = font_manager.FontProperties(fname=path) 107 | except: 108 | print('No font in location, using default font.') 109 | prop=None 110 | types=['CHOSUNG','JUNGSUNG','JONGSUNG'] 111 | 112 | confusion_list={'CHOSUNG':0,'JUNGSUNG':0,'JONGSUNG':0} 113 | index_list={'CHOSUNG':korean_manager.CHOSUNG_LIST,'JUNGSUNG':korean_manager.JUNGSUNG_LIST,'JONGSUNG':korean_manager.JONGSUNG_LIST} 114 | file_list=fnmatch.filter(os.listdir(args.data_path), f'{key_text}*.pickle') 115 | for x in progressbar.progressbar(file_list): 116 | #Read pickle 117 | path=os.path.join(args.data_path,x) 118 | with open(path,'rb') as handle: 119 | data=pickle.load(handle) 120 | #Predict image 121 | pred=model.model.predict(data['image']) 122 | pred_dict={} 123 | for idx,t in enumerate(types): 124 | pred_dict[t]=np.argmax(pred[idx],axis=1) 125 | 126 | cho,jung,jong=korean_manager.korean_split_numpy(data['label']) 127 | truth_label={'CHOSUNG':cho, 'JUNGSUNG':jung,'JONGSUNG':jong} 128 | #Add to confusion_matrix 129 | for t in types: 130 | labels=range(len(index_list[t])) 131 | confusion_list[t]=confusion_matrix(truth_label[t],pred_dict[t],labels=labels)+confusion_list[t] 132 | 133 | #Plot confusion matrix using seaborn 134 | 135 | for t in types: 136 | df_cm = pd.DataFrame(confusion_list[t], index=index_list[t], columns=index_list[t]) 137 | 138 | sn.set(rc={'figure.figsize':(20,18)}) 139 | ax = sn.heatmap(df_cm, cmap='Oranges', annot=True, fontproperties=prop) 140 | fig = ax.get_figure() 141 | fig.savefig(os.path.join('./logs','confusion_matrix_'+key_text+'_'+t+".png")) 142 | plt.clf() 143 | 144 | def evaluate(model,key_text,plot_wrong=True): 145 | #Evaluate top-n accuracy 146 | correct_num,total_num=0,0 147 | file_list=fnmatch.filter(os.listdir(args.data_path), f'{key_text}*.pickle') 148 | 149 | wrong_list=[] 150 | for x in progressbar.progressbar(file_list): 151 | #Read pickle 152 | path=os.path.join(args.data_path,x) 153 | with open(path,'rb') as handle: 154 | data=pickle.load(handle) 155 | #Predict image 156 | pred=model.predict(data['image'],n=args.top_n) 157 | #Compare/calculate acc 158 | for idx,pred_ in enumerate(pred): 159 | total_num+=1 160 | if data['label'][idx] in pred_: 161 | correct_num+=1 162 | else: 163 | wrong_list.append((data['image'][idx],data['label'][idx])) 164 | 165 | if plot_wrong: 166 | fig = plt.figure(figsize=(10,10)) 167 | for x in range(100): 168 | plt.subplot(x,10,10) 169 | plt.imshow(wrong_list[x][0]) 170 | plt.xlabel('Pred:'+str(wrong_list[x][1])) 171 | plt.savefig(f'./logs/{key_text}_Wrong_examples.png') 172 | plt.clf() 173 | return 100*correct_num/total_num 174 | 175 | if __name__=='__main__': 176 | args = parser.parse_args() 177 | plt.rc('font', family='NanumBarunGothic') 178 | 179 | KoOCR=model.KoOCR(split_components=args.split_components,weight_path=args.weights) 180 | 181 | if args.accuracy: 182 | acc=evaluate(KoOCR,'handwritten') 183 | print('Handwritten OCR Accuracy:',acc) 184 | acc=evaluate(KoOCR,'printed') 185 | print('Printed OCR Accuracy:',acc) 186 | 187 | if args.confusion_matrix: 188 | generate_confusion_matrix(KoOCR,'handwritten') 189 | print('Handwritten confusion matrix generated.') 190 | generate_confusion_matrix(KoOCR,'printed') 191 | print('Printed confusion matrix generated.') 192 | 193 | if args.class_activation: 194 | generate_CAM(KoOCR,'handwritten') 195 | print('Handwritten CAM image generated.') 196 | generate_CAM(KoOCR,'printed') 197 | print('Printed CAM image generated.') 198 | 199 | if args.show_augmentation: 200 | plot_augmentation() -------------------------------------------------------------------------------- /files/batang.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/files/batang.ttf -------------------------------------------------------------------------------- /files/credentials.json: -------------------------------------------------------------------------------- 1 | {"installed":{"client_id":"813774342851-v8j21ocifp4p03nhh6d28arro9rtacok.apps.googleusercontent.com","project_id":"koocr-1612166115153","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"4-VwR7EiiQnjl94_4FLh7pn6","redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"]}} -------------------------------------------------------------------------------- /files/hangeul_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/files/hangeul_image.png -------------------------------------------------------------------------------- /files/requirements.txt: -------------------------------------------------------------------------------- 1 | progressbar2 2 | googledrivedownloader 3 | tensorflow 4 | numpy 5 | matplotlib 6 | -------------------------------------------------------------------------------- /files/ttf_links.txt: -------------------------------------------------------------------------------- 1 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 가람연꽃.ttf 2 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 갈맷글.ttf 3 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 강부장님체.ttf 4 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 강인한 위로.ttf 5 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 고딕 아니고 고딩.ttf 6 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 고려글꼴.ttf 7 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 곰신체.ttf 8 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 규리의 일기.ttf 9 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 금은보화.ttf 10 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 기쁨밝음.ttf 11 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 김유이체.ttf 12 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 꽃내음.ttf 13 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 끄트머리체.ttf 14 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 나는 이겨낸다.ttf 15 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 나무정원.ttf 16 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 나의 아내 손글씨.ttf 17 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 노력하는 동희.ttf 18 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 느릿느릿체.ttf 19 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 다시 시작해.ttf 20 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 다진체.ttf 21 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 다채사랑.ttf 22 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 다행체.ttf 23 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 달의궤도.ttf 24 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 대광유리.ttf 25 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 대한민국 열사체.ttf 26 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 동화또박.ttf 27 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 둥근인연.ttf 28 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 따뜻한 작별.ttf 29 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 따악단단.ttf 30 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 딸에게 엄마가.ttf 31 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 가람연꽃.ttf 32 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 갈맷글.ttf 33 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 강부장님체.ttf 34 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 강인한 위로.ttf 35 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 고딕 아니고 고딩.ttf 36 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 고려글꼴.ttf 37 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 곰신체.ttf 38 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 규리의 일기.ttf 39 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 금은보화.ttf 40 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 기쁨밝음.ttf 41 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 김유이체.ttf 42 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 꽃내음.ttf 43 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 끄트머리체.ttf 44 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 나는 이겨낸다.ttf 45 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 나무정원.ttf 46 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 나의 아내 손글씨.ttf 47 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 노력하는 동희.ttf 48 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 느릿느릿체.ttf 49 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 다시 시작해.ttf 50 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 다진체.ttf 51 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 다채사랑.ttf 52 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 다행체.ttf 53 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 달의궤도.ttf 54 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 대광유리.ttf 55 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 대한민국 열사체.ttf 56 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 동화또박.ttf 57 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 둥근인연.ttf 58 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 따뜻한 작별.ttf 59 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 따악단단.ttf 60 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 딸에게 엄마가.ttf 61 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 또박또박.ttf 62 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 마고체.ttf 63 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 맛있는체.ttf 64 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 몽돌.ttf 65 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 무궁화.ttf 66 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 무진장체.ttf 67 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 미니 손글씨.ttf 68 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 미래나무.ttf 69 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download_1008/나눔손글씨 바른정신.ttf 70 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 바른히피.ttf 71 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 반짝반짝 별.ttf 72 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 배은혜체.ttf 73 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 백의의 천사.ttf 74 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 버드나무.ttf 75 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 범솜체.ttf 76 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 부장님 눈치체.ttf 77 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 북극성.ttf 78 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 비상체.ttf 79 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 빵구니맘 손글씨.ttf 80 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 사랑해 아들.ttf 81 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 상해찬미체.ttf 82 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 성실체.ttf 83 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 세계적인 한글.ttf 84 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 세아체.ttf 85 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 세화체.ttf 86 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 소미체.ttf 87 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 소방관의 기도.ttf 88 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 손편지체.ttf 89 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 수줍은 대학생.ttf 90 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 시우 귀여워.ttf 91 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download_1015/나눔손글씨 신혼부부.ttf 92 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 아기사랑체.ttf 93 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 아름드리 꽃나무.ttf 94 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 아빠글씨.ttf 95 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 아빠의 연애편지.ttf 96 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 아인맘 손글씨.ttf 97 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 아줌마 자유.ttf 98 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 안쌍체.ttf 99 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 암스테르담.ttf 100 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 야근하는 김주임.ttf 101 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 야채장수 백금례.ttf 102 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 엄마사랑.ttf 103 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 엉겅퀴체.ttf 104 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 여름글씨.ttf 105 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 연지체.ttf 106 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 열아홉의 반짝임.ttf 107 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 열일체.ttf 108 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 예당체.ttf 109 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 예쁜 민경체.ttf 110 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 옥비체.ttf 111 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 와일드.ttf 112 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download_1008/나눔손글씨 외할머니글씨.ttf 113 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 왼손잡이도 예뻐.ttf 114 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 우리딸 손글씨.ttf 115 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 유니 띵땅띵땅.ttf 116 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 의미있는 한글.ttf 117 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 자부심지우.ttf 118 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 잘하고 있어.ttf 119 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 장미체.ttf 120 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 점꼴체.ttf 121 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 정은체.ttf 122 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 중학생.ttf 123 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 진주 박경아체.ttf 124 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 철필글씨.ttf 125 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 초딩희망.ttf 126 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download_1008/나눔손글씨 칼국수.ttf 127 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 코코체.ttf 128 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 하나되어 손글씨.ttf 129 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 하나손글씨.ttf 130 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 하람체.ttf 131 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 한윤체.ttf 132 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 할아버지의나눔.ttf 133 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 행복한 도비.ttf 134 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 혁이체.ttf 135 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 혜준체.ttf 136 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 효남 늘 화이팅.ttf 137 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 희망누리.ttf 138 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download_200310/나눔손글씨 흰꼬리수리.ttf 139 | https://ssl.pstatic.net/static/clova/service/clova_ai/event/handwriting/download/나눔손글씨 힘내라는 말보단.ttf 140 | -------------------------------------------------------------------------------- /load_data.py: -------------------------------------------------------------------------------- 1 | from google_drive_downloader import GoogleDriveDownloader as gdd 2 | import concurrent.futures 3 | import os 4 | import zipfile 5 | from py7zr import unpack_7zarchive 6 | import shutil 7 | 8 | import argparse 9 | 10 | 11 | def str2bool(v): 12 | if isinstance(v, bool): 13 | return v 14 | if v.lower() in ('yes', 'true', 't', 'y', '1'): 15 | return True 16 | elif v.lower() in ('no', 'false', 'f', 'n', '0'): 17 | return False 18 | else: 19 | raise argparse.ArgumentTypeError('Boolean value expected.') 20 | 21 | parser = argparse.ArgumentParser(description='Download dataset') 22 | parser.add_argument("--sevenzip", type=str2bool,default=True) 23 | 24 | def unzip_7z(): 25 | val_data_7z_link='17NusZQw2RKpBIvCKp6hW6RuJKY43SWqz' 26 | data_7z_link='1-I3BtCzYE7swpKERGygRhlMnim2xX0_2' 27 | 28 | print("Downloading data...") 29 | 30 | gdd.download_file_from_google_drive(file_id=data_7z_link,dest_path=os.path.join(data_path,'data.7z'),unzip=False) 31 | gdd.download_file_from_google_drive(file_id=val_data_7z_link,dest_path=os.path.join(val_data_path,'val_data.7z'),unzip=False) 32 | 33 | print('Unzipping data...') 34 | shutil.register_unpack_format('7zip', ['.7z'], unpack_7zarchive) 35 | 36 | shutil.unpack_archive(os.path.join(data_path,'data.7z'), data_path) 37 | os.remove(os.path.join(data_path,'data.7z')) 38 | 39 | shutil.unpack_archive(os.path.join(val_data_path,'val_data.7z'), val_data_path) 40 | os.remove(os.path.join(val_data_path,'val_data.7z')) 41 | print("Downloading complete...") 42 | 43 | def unzip_zip(): 44 | val_data_link='1WOP_sQsu4vXCY739VGgiWIbHyOozcjHw' 45 | data_link='1HBu43eBO-vXJsJp8crEp_Iih3_U7QSR2' 46 | 47 | print("Downloading data...") 48 | gdd.download_file_from_google_drive(file_id=data_link,dest_path=os.path.join(data_path,'data.zip'),unzip=False) 49 | gdd.download_file_from_google_drive(file_id=val_data_link,dest_path=os.path.join(val_data_path,'val_data.zip'),unzip=False) 50 | 51 | print('Unzipping data...') 52 | with zipfile.ZipFile(os.path.join(data_path,'data.zip'), 'r') as zip_ref: 53 | zip_ref.extractall(data_path) 54 | os.remove(os.path.join(data_path,'data.zip')) 55 | 56 | with zipfile.ZipFile(os.path.join(val_data_path,'val_data.zip'), 'r') as zip_ref: 57 | zip_ref.extractall(val_data_path) 58 | os.remove(os.path.join(val_data_path,'val_data.zip')) 59 | print("Downloading complete...") 60 | 61 | if __name__ =='__main__': 62 | args = parser.parse_args() 63 | 64 | val_data_path='./val_data' 65 | data_path='./data' 66 | #Create data directory 67 | if os.path.isdir(val_data_path)==False: 68 | os.makedirs(val_data_path) 69 | if os.path.isdir(data_path)==False: 70 | os.makedirs(data_path) 71 | 72 | if args.sevenzip: 73 | unzip_7z() 74 | else: 75 | unzip_zip() -------------------------------------------------------------------------------- /logs/debug.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/logs/debug.txt -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | import dataset 5 | import utils.korean_manager as korean_manager 6 | from PIL import Image 7 | import random 8 | import os 9 | from IPython.display import clear_output 10 | import gc 11 | import wandb 12 | import datetime 13 | import shutil 14 | from tqdm import tqdm 15 | from keras_adabound import AdaBound 16 | import utils.predict_char as predict_char 17 | from utils.model_architectures import VGG16,InceptionResnetV2,MobilenetV3,EfficientCNN,AFL_Model 18 | from utils.MelnykNet import melnyk_net 19 | class KoOCR(): 20 | def __init__(self,split_components=True,weight_path='',fc_link='',network_type='melnyk',image_size=96,direct_map=False,refinement_t=4,\ 21 | iterative_refinement=False,data_augmentation=False,adversarial_learning=False): 22 | self.split_components=split_components 23 | self.iterative_refinement=iterative_refinement 24 | self.refinement_t=refinement_t 25 | self.charset=korean_manager.load_charset() 26 | self.adversarial_learning=adversarial_learning 27 | #Build and load model 28 | if weight_path: 29 | self.model = tf.keras.models.load_model(weight_path,compile=False) 30 | else: 31 | model_list={'VGG16':VGG16,'inception-resnet':InceptionResnetV2,'mobilenet':MobilenetV3,'efficient-net':EfficientCNN,'melnyk':melnyk_net, 32 | 'afl':AFL_Model} 33 | settings={'split_components':split_components,'input_shape':image_size,'direct_map':direct_map,'fc_link':fc_link,'refinement_t':refinement_t,\ 34 | 'iterative_refinement':iterative_refinement,'data_augmentation':data_augmentation,'adversarial_learning':adversarial_learning} 35 | self.model=model_list[network_type](settings) 36 | if iterative_refinement: 37 | self.decoders=self.find_decoders() 38 | 39 | def find_decoders(self): 40 | return 0 41 | 42 | def predict(self,image,n=1): 43 | if self.split_components: 44 | return predict_char.predict_split(self.model,image,n) 45 | else: 46 | return predict_char.predict_complete(self.model,image,n) 47 | 48 | def plot_val_image(self,val_data): 49 | #Load validation data 50 | val_x,val_y=val_data 51 | #Predict classes 52 | indicies=random.sample(range(len(val_x)),10) 53 | val_x=val_x[indicies] 54 | pred_y=self.predict(val_x,10) 55 | 56 | fig = plt.figure(figsize=(10,1)) 57 | for idx in range(10): 58 | plt.subplot(1,10,idx+1) 59 | plt.imshow(val_x[idx],cmap='gray') 60 | plt.axis('off') 61 | plt.savefig('./logs/image.png') 62 | print(pred_y) 63 | 64 | def compile_adversarial_model(self,lr,opt,adversarial_ratio=0): 65 | #build adversarial model for training 66 | input_image=self.model.input 67 | disc_output=self.model.get_layer('DISC') 68 | 69 | self.discriminator=tf.keras.models.Model(self.model.input,disc_output.output) 70 | 71 | for l in self.model.layers: 72 | l.trainable=False 73 | 74 | self.model.get_layer('disc_start').trainable=True 75 | self.model.get_layer('DISC').trainable=True 76 | 77 | lr=lr*adversarial_ratio*3 78 | if opt =='sgd': 79 | optimizer=tf.keras.optimizers.SGD(lr) 80 | elif opt=='adam': 81 | optimizer=tf.keras.optimizers.Adam(lr) 82 | elif opt=='adabound': 83 | optimizer=AdaBound(lr=lr,final_lr=lr*100) 84 | 85 | self.discriminator.compile(optimizer=optimizer,loss='binary_crossentropy') 86 | 87 | def compile_model(self,lr,opt,adversarial_ratio=0): 88 | def inverse_bce(y_true,y_pred): 89 | y_true=y_true*-1+1 90 | return tf.keras.losses.binary_crossentropy(y_true,y_pred) 91 | 92 | #Compile model 93 | if opt =='sgd': 94 | optimizer=tf.keras.optimizers.SGD(lr) 95 | elif opt=='adam': 96 | optimizer=tf.keras.optimizers.Adam(lr) 97 | elif opt=='adabound': 98 | optimizer=AdaBound(lr=lr,final_lr=lr*100) 99 | 100 | if self.iterative_refinement: 101 | losses="categorical_crossentropy" 102 | elif self.split_components: 103 | if self.adversarial_learning: 104 | losses = { 105 | "CHOSUNG": "categorical_crossentropy", 106 | "JUNGSUNG": "categorical_crossentropy", 107 | "JONGSUNG": "categorical_crossentropy", 108 | 'DISC':inverse_bce} 109 | if self.fit_discriminator: 110 | lossWeights = {"CHOSUNG": 1.0-adversarial_ratio, "JUNGSUNG": 1.0-adversarial_ratio, 111 | "JONGSUNG":1.0-adversarial_ratio,"DISC":3*adversarial_ratio} 112 | else: 113 | lossWeights = {"CHOSUNG": 1.0, "JUNGSUNG": 1.0,"JONGSUNG":1.0,"DISC":0} 114 | else: 115 | losses = { 116 | "CHOSUNG": "categorical_crossentropy", 117 | "JUNGSUNG": "categorical_crossentropy", 118 | "JONGSUNG": "categorical_crossentropy"} 119 | lossWeights = {"CHOSUNG": 1.0, "JUNGSUNG": 1.0,"JONGSUNG":1.0} 120 | else: 121 | losses="categorical_crossentropy" 122 | lossWeights=None 123 | 124 | if self.adversarial_learning: 125 | self.model.trainable=True 126 | self.model.get_layer('disc_start').trainable=False 127 | self.model.get_layer('DISC').trainable=False 128 | 129 | self.model.compile(optimizer=optimizer, loss=losses,metrics=["accuracy"],loss_weights=lossWeights) 130 | 131 | def fit_adversarial(self,train_x,train_y,val_x,val_y,batch_size): 132 | train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).batch(batch_size) 133 | loss_arr,total_p=0,0 134 | 135 | if self.verbose==1: 136 | pbar=tqdm(train_dataset) 137 | else: 138 | pbar=train_dataset 139 | for image,label in pbar: 140 | out=self.model.train_on_batch(image,label) 141 | loss_arr+=np.array(out) 142 | total_p+=1 143 | 144 | if self.fit_discriminator: 145 | self.discriminator.train_on_batch(image,label['DISC']) 146 | results = self.model.evaluate(val_x, val_y, batch_size=128,verbose=self.verbose) 147 | print("Training L:", list(loss_arr/total_p)) 148 | 149 | loss_dict = {name+'_loss': pred for name, pred in zip(self.model.output_names, out[:len(out)//2])} 150 | acc_dict = {name+'_accuracy': pred for name, pred in zip(self.model.output_names, out[len(out)//2:])} 151 | z = {**loss_dict, **acc_dict} 152 | return z 153 | 154 | def train(self,epochs=10,lr=0.001,data_path='./data',patch_size=10,batch_size=32,optimizer='adabound',zip_weights=False, 155 | adversarial_ratio=0.15,log_tensorboard=True,log_wandb=False,setup_wandb=False,fit_discriminator=True,silent_mode=False): 156 | def write_tensorboard(summary_writer,history,step): 157 | with summary_writer.as_default(): 158 | if self.split_components: 159 | tf.summary.scalar('training_loss', history.history['loss'][0], step=step) 160 | tf.summary.scalar('CHOSUNG_accuracy', history.history['CHOSUNG_accuracy'][0], step=step) 161 | tf.summary.scalar('JUNGSUNG_accuracy', history.history['JUNGSUNG_accuracy'][0], step=step) 162 | tf.summary.scalar('JONGSUNG_accuracy', history.history['JONGSUNG_accuracy'][0], step=step) 163 | 164 | tf.summary.scalar('val_loss', history.history['val_loss'][0], step=step) 165 | tf.summary.scalar('val_CHOSUNG_accuracy', history.history['val_CHOSUNG_accuracy'][0], step=step) 166 | tf.summary.scalar('val_JUNGSUNG_accuracy', history.history['val_JUNGSUNG_accuracy'][0], step=step) 167 | tf.summary.scalar('val_JONGSUNG_accuracy', history.history['val_JONGSUNG_accuracy'][0], step=step) 168 | else: 169 | tf.summary.scalar('training_loss', history.history['loss'][0], step=step) 170 | tf.summary.scalar('val_loss', history.history['accuracy'][0], step=step) 171 | tf.summary.scalar('training_accuracy', history.history['val_loss'][0], step=step) 172 | tf.summary.scalar('val_accuracy', history.history['val_accuracy'][0], step=step) 173 | 174 | def setup_wandboard(): 175 | wandb.init(project="KoOCR", config={ 176 | 'AFL': self.adversarial_learning, 177 | 'Iterative Refinement': self.iteratve_refinement, 178 | "optiminzer": optimizer, 179 | "batch_size": batch_size, 180 | 'learning_rate':lr, 181 | 'AFL ratio':adversarial_ratio, 182 | 'Split components':self.split_components 183 | }) 184 | def write_wandb(history): 185 | wandb.log(history) 186 | train_dataset=dataset.DataPickleLoader(split_components=self.split_components,data_path=data_path,patch_size=patch_size, 187 | return_image_type=self.adversarial_learning,silent_mode=silent_mode) 188 | val_x,val_y=train_dataset.get_val() 189 | if self.iterative_refinement: 190 | val_y=[val_y['CHOSUNG'],val_y['JUNGSUNG'],val_y['JONGSUNG']]*self.refinement_t 191 | self.fit_discriminator=fit_discriminator 192 | self.compile_model(lr,optimizer,adversarial_ratio) 193 | if self.adversarial_learning: 194 | self.compile_adversarial_model(lr,optimizer,adversarial_ratio) 195 | 196 | summary_writer = tf.summary.create_file_writer("./logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")) 197 | if setup_wandb: 198 | setup_wandboard() 199 | step=0 200 | 201 | if silent_mode: 202 | self.verbose=2 203 | else: 204 | self.verbose=1 205 | 206 | for epoch in range(epochs): 207 | print('Training epoch',epoch) 208 | self.plot_val_image(val_data=(val_x,val_y)) 209 | epoch_end=False 210 | while epoch_end==False: 211 | #Train on loaded dataset batch 212 | train_x,train_y,epoch_end=train_dataset.get() 213 | if self.iterative_refinement: 214 | train_y=[train_y['CHOSUNG'],train_y['JUNGSUNG'],train_y['JONGSUNG']]*self.refinement_t 215 | 216 | if self.adversarial_learning: 217 | history=self.fit_adversarial(train_x,train_y,val_x,val_y,batch_size) 218 | else: 219 | history=self.model.fit(x=train_x,y=train_y,epochs=1,validation_data=(val_x,val_y),batch_size=batch_size,verbose=self.verbose) 220 | #Log losses to Tensorboard 221 | if log_tensorboard: 222 | write_tensorboard(summary_writer,history,step) 223 | if log_wandb: 224 | write_wandb(history) 225 | step+=1 226 | #Clear garbage memory 227 | tf.keras.backend.clear_session() 228 | gc.collect() 229 | 230 | #Save weights in checkpoint 231 | self.model.save('./logs/weights', save_format='tf') 232 | if zip_weights: 233 | shutil.make_archive('weights_epoch_'+str(epoch), 'zip', './logs/weights') -------------------------------------------------------------------------------- /pretrained/plain_melnyk/saved_model.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/pretrained/plain_melnyk/saved_model.pb -------------------------------------------------------------------------------- /pretrained/plain_melnyk/variables/variables.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/pretrained/plain_melnyk/variables/variables.data-00000-of-00001 -------------------------------------------------------------------------------- /pretrained/plain_melnyk/variables/variables.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/pretrained/plain_melnyk/variables/variables.index -------------------------------------------------------------------------------- /pretrained/plain_melnyk_evaluation/CAM_handwritten.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/pretrained/plain_melnyk_evaluation/CAM_handwritten.png -------------------------------------------------------------------------------- /pretrained/plain_melnyk_evaluation/CAM_printed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/pretrained/plain_melnyk_evaluation/CAM_printed.png -------------------------------------------------------------------------------- /pretrained/plain_melnyk_evaluation/handwritten_CHOSUNG.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/pretrained/plain_melnyk_evaluation/handwritten_CHOSUNG.png -------------------------------------------------------------------------------- /pretrained/plain_melnyk_evaluation/handwritten_JONGSUNG.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/pretrained/plain_melnyk_evaluation/handwritten_JONGSUNG.png -------------------------------------------------------------------------------- /pretrained/plain_melnyk_evaluation/handwritten_JUNGSUNG.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/pretrained/plain_melnyk_evaluation/handwritten_JUNGSUNG.png -------------------------------------------------------------------------------- /pretrained/plain_melnyk_evaluation/printed_CHOSUNG.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/pretrained/plain_melnyk_evaluation/printed_CHOSUNG.png -------------------------------------------------------------------------------- /pretrained/plain_melnyk_evaluation/printed_JONGSUNG.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/pretrained/plain_melnyk_evaluation/printed_JONGSUNG.png -------------------------------------------------------------------------------- /pretrained/plain_melnyk_evaluation/printed_JUNGSUNG.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sieu-n/KoOCR-tensorflow/e9d14bf1988441a8da2a7a91ddf5cb9be851af72/pretrained/plain_melnyk_evaluation/printed_JUNGSUNG.png -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import model 2 | import dataset 3 | import argparse 4 | import os 5 | 6 | #bool type for arguments 7 | def str2bool(v): 8 | if isinstance(v, bool): 9 | return v 10 | if v.lower() in ('yes', 'true', 't', 'y', '1'): 11 | return True 12 | elif v.lower() in ('no', 'false', 'f', 'n', '0'): 13 | return False 14 | else: 15 | raise argparse.ArgumentTypeError('Boolean value expected.') 16 | #Define arguments 17 | parser = argparse.ArgumentParser(description='Download dataset') 18 | parser.add_argument("--data_path", type=str,default='./data') 19 | parser.add_argument("--image_size", type=int,default=256) 20 | parser.add_argument("--split_components", type=str2bool,default=True) 21 | parser.add_argument("--patch_size", type=int,default=10) 22 | parser.add_argument("--zip_weights", type=str2bool,default=False) 23 | parser.add_argument("--network", type=str,default='melnyk',choices=['VGG16','inception-resnet','mobilenet','efficient-net','melnyk','afl']) 24 | parser.add_argument("--fc_link", type=str,default='',choices=['', 'GAP','GWAP','GWOAP']) 25 | parser.add_argument("--iterative_refinement", type=str2bool,default=False) 26 | parser.add_argument("--refinement_t", type=int,default=4) 27 | parser.add_argument("--data_augmentation", type=str2bool,default=False) 28 | parser.add_argument("--fit_discriminator", type=str2bool,default=True) 29 | parser.add_argument("--adversarial_learning", type=str2bool,default=False) 30 | parser.add_argument("--adversarial_ratio", type=float,default=0.15) 31 | parser.add_argument("--silent_mode", type=str2bool,default=False) 32 | 33 | parser.add_argument("--log_tensorboard", type=str2bool,default=True) 34 | parser.add_argument("--log_wandb", type=str2bool,default=False) 35 | parser.add_argument("--setup_wandb", type=str2bool,default=False) 36 | 37 | parser.add_argument("--optimizer", type=str,default='adabound',choices=['sgd', 'adam','adabound']) 38 | parser.add_argument("--direct_map", type=str2bool,default=False) 39 | parser.add_argument("--batch_size", type=int,default=32) 40 | parser.add_argument("--epochs", type=int,default=50) 41 | parser.add_argument("--weights", type=str,default='') 42 | parser.add_argument("--learning_rate", type=float,default=0.001) 43 | 44 | if __name__=='__main__': 45 | args = parser.parse_args() 46 | 47 | KoOCR=model.KoOCR(split_components=args.split_components,weight_path=args.weights,fc_link=args.fc_link,iterative_refinement=args.iterative_refinement,\ 48 | network_type=args.network,image_size=args.image_size,direct_map=args.direct_map,refinement_t=args.refinement_t,data_augmentation=args.data_augmentation, 49 | adversarial_learning=args.adversarial_learning) 50 | KoOCR.train(epochs=args.epochs,lr=args.learning_rate,data_path=args.data_path,patch_size=args.patch_size,batch_size=args.batch_size,optimizer=args.optimizer, 51 | zip_weights=args.zip_weights,adversarial_ratio=args.adversarial_ratio,log_tensorboard=args.log_tensorboard,log_wandb=args.log_wandb, 52 | setup_wandb=args.setup_wandb,fit_discriminator=args.fit_discriminator,silent_mode=args.silent_mode) -------------------------------------------------------------------------------- /utils/CustomLayers.py: -------------------------------------------------------------------------------- 1 | from tensorflow.keras.models import Model 2 | import tensorflow as tf 3 | import numpy as np 4 | import cv2 5 | import utils.korean_manager as korean_manager 6 | 7 | class BahdanauAttention(tf.keras.Model): 8 | def __init__(self, units): 9 | super(BahdanauAttention, self).__init__() 10 | self.W1 = tf.keras.layers.Dense(units) 11 | self.W2 = tf.keras.layers.Dense(units) 12 | self.V = tf.keras.layers.Dense(1) 13 | 14 | def __call__(self, features, hidden): 15 | # features(CNN_encoder output) shape == (batch_size, 36, embedding_dim) 16 | 17 | # hidden shape == (batch_size, hidden_size) 18 | # hidden_with_time_axis shape == (batch_size, 1, hidden_size) 19 | hidden_with_time_axis = tf.expand_dims(hidden, 1) 20 | 21 | # attention_hidden_layer shape == (batch_size, 36, units) 22 | attention_hidden_layer = (tf.nn.tanh(self.W1(features) + 23 | self.W2(hidden_with_time_axis))) 24 | 25 | # score shape == (batch_size, 36, 1) 26 | # This gives you an unnormalized score for each image feature. 27 | score = self.V(attention_hidden_layer) 28 | 29 | # attention_weights shape == (batch_size, 36, 1) 30 | attention_weights = tf.nn.softmax(score, axis=1) 31 | 32 | # context_vector shape after sum == (batch_size, hidden_size) 33 | context_vector = attention_weights * features 34 | context_vector = tf.reduce_sum(context_vector, axis=1) 35 | 36 | return context_vector, attention_weights 37 | 38 | class RNN_Decoder(tf.keras.Model): 39 | def __init__(self, units, class_types): 40 | #units: # classes 41 | super(RNN_Decoder, self).__init__() 42 | self.units = units 43 | self.gru = tf.keras.layers.GRU(self.units, 44 | return_sequences=True, 45 | return_state=True, 46 | recurrent_initializer='glorot_uniform') 47 | self.fc1 = tf.keras.layers.Dense(self.units) 48 | self.fc2 = tf.keras.layers.Dense(class_types) 49 | 50 | self.attention = BahdanauAttention(self.units) 51 | 52 | def __call__(self, features, hidden): 53 | # hidden: previous states features: feature map(conv output) 54 | # defining attention as a separate model 55 | 56 | context_vector, attention_weights = self.attention(features, hidden) 57 | 58 | # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size) 59 | x = tf.expand_dims(context_vector, 1) 60 | 61 | # passing the concatenated vector to the GRU 62 | output, state = self.gru(x) 63 | 64 | # shape == (batch_size, max_length, hidden_size) 65 | x = self.fc1(output) 66 | 67 | # x shape == (batch_size * max_length, hidden_size) 68 | x = tf.reshape(x, (-1, x.shape[2])) 69 | 70 | # output shape == (batch_size * max_length, class_types) 71 | x = self.fc2(x) 72 | 73 | return x, state, attention_weights 74 | 75 | class GlobalWeightedAveragePooling(tf.keras.Model): 76 | #Implementation of GlobalWeightedAveragePooling 77 | def __init__(self,kernel_initializer=tf.keras.initializers.RandomNormal(stddev=0.01),**kwargs): 78 | #self.num_outputs = num_outputs 79 | super(GlobalWeightedAveragePooling, self).__init__(**kwargs) 80 | self.kernel_initializer =kernel_initializer 81 | 82 | 83 | def build(self, input_shape): 84 | #input_shape=(w,h,c) 85 | self.kernel = self.add_weight("kernel",shape=input_shape[1:],initializer=self.kernel_initializer) 86 | 87 | def call(self, input): 88 | com=tf.math.multiply(input, self.kernel) 89 | return tf.math.reduce_sum(com,axis=[1,2]) 90 | 91 | class GlobalWeightedOutputAveragePooling(tf.keras.layers.Layer): 92 | def __init__(self): 93 | #self.num_outputs = num_outputs 94 | super(GlobalWeightedOutputAveragePooling, self).__init__() 95 | 96 | def build(self, input_shape): 97 | #input_shape=(w,h,c) 98 | self.kernel = self.add_weight("kernel",shape=(input_shape[-1],)) 99 | 100 | def call(self, input): 101 | out=tf.keras.layers.GlobalAveragePooling2D()(input) 102 | return tf.math.multiply(input, self.kernel) 103 | 104 | class MultiOutputGradCAM: 105 | #CAM Implementaion referenced from https://www.pyimagesearch.com/2020/03/09/grad-cam-visualize-class-activation-maps-with-keras-tensorflow-and-deep-learning/ 106 | def __init__(self, model, character): 107 | # store the model, the class index used to measure the class 108 | # activation map, and the layer to be used when visualizing 109 | # the class activation map 110 | self.model = model 111 | self.character_Idx= korean_manager.korean_split(character) 112 | # if the layer name is None, attempt to automatically find 113 | # the target output layer 114 | self.layerName = self.find_target_layer() 115 | 116 | def find_target_layer(self): 117 | # attempt to find the final convolutional layer in the network 118 | # by looping over the layers of the network in reverse order 119 | for layer in reversed(self.model.layers): 120 | # check to see if the layer has a 4D output 121 | if len(layer.output_shape) == 4: 122 | return layer.name 123 | # otherwise, we could not find a 4D layer so the GradCAM 124 | # algorithm cannot be applied 125 | raise ValueError("Could not find 4D layer. Cannot apply GradCAM.") 126 | 127 | def compute_heatmap(self, image, eps=1e-8): 128 | # construct our gradient model by supplying (1) the inputs 129 | # to our pre-trained model, (2) the output of the (presumably) 130 | # final 4D layer in the network, and (3) the output of the 131 | # softmax activations from the model 132 | gradModel = Model(inputs=[self.model.inputs],\ 133 | outputs=[self.model.get_layer(self.layerName).output]+[self.model.output]) 134 | image=image.reshape(1,image.shape[0],image.shape[1],1) 135 | heatmap_list=[] 136 | 137 | for component in range(3): 138 | # record operations for automatic differentiation 139 | with tf.GradientTape() as tape: 140 | # cast the image tensor to a float-32 data type, pass the 141 | # image through the gradient model, and grab the loss 142 | # associated with the specific class index 143 | inputs = tf.cast(image, tf.float32) 144 | (convOutputs, predictions) = gradModel(inputs) 145 | loss=predictions[component][:, self.character_Idx[component]] 146 | # use automatic differentiation to compute the gradients 147 | grads = tape.gradient(loss, convOutputs) 148 | # compute the guided gradients 149 | castConvOutputs = tf.cast(convOutputs > 0, "float32") 150 | castGrads = tf.cast(grads > 0, "float32") 151 | guidedGrads = castConvOutputs * castGrads * grads 152 | # the convolution and guided gradients have a batch dimension 153 | # (which we don't need) so let's grab the volume itself and 154 | # discard the batch 155 | convOutputs = convOutputs[0] 156 | guidedGrads = guidedGrads[0] 157 | # compute the average of the gradient values, and using them 158 | # as weights, compute the ponderation of the filters with 159 | # respect to the weights 160 | weights = tf.reduce_mean(guidedGrads, axis=(0, 1)) 161 | cam = tf.reduce_sum(tf.multiply(weights, convOutputs), axis=-1).numpy() 162 | cam=cam.reshape(cam.shape[0],cam.shape[1],1) 163 | # reshape 164 | (w, h) = (image.shape[1], image.shape[2]) 165 | heatmap = tf.image.resize(cam, (w, h)).numpy() 166 | # normalize the heatmap such that all values lie in the range 167 | # [0, 1], scale the resulting values to the range [0, 255], 168 | # and then convert to an unsigned 8-bit integer 169 | numer = heatmap - np.min(heatmap) 170 | denom = (heatmap.max() - heatmap.min()) + eps 171 | heatmap = numer / denom 172 | heatmap = (heatmap * 255).astype("uint8") 173 | 174 | heatmap_list.append(heatmap) 175 | return heatmap_list -------------------------------------------------------------------------------- /utils/MelnykNet.py: -------------------------------------------------------------------------------- 1 | #Referenced from original implementation of Melnyk-net: https://github.com/pavlo-melnyk/offline-HCCR/blob/master/src/melnyk_net.py 2 | import os 3 | 4 | import numpy as np 5 | 6 | from tensorflow.keras.layers import Input, Dense, Dropout, Flatten, Reshape, GlobalAveragePooling2D, Activation, BatchNormalization, Conv2D, MaxPooling2D, AveragePooling2D 7 | from tensorflow.keras.models import Model 8 | from tensorflow.keras.regularizers import l2 9 | from tensorflow.keras.optimizers import SGD 10 | from tensorflow.keras.initializers import RandomNormal 11 | 12 | from utils.CustomLayers import GlobalWeightedAveragePooling 13 | from utils.model_components import build_FC,PreprocessingPipeline 14 | 15 | def melnyk_net(settings): 16 | random_normal = RandomNormal(stddev=0.001) 17 | reg=0 18 | 19 | input_image=Input(shape=(settings['input_shape'],settings['input_shape']),name='input_image') 20 | preprocessed=Reshape((settings['input_shape'],settings['input_shape'],1))(input_image) 21 | preprocessed=PreprocessingPipeline(settings['direct_map'],settings['data_augmentation'])(preprocessed) 22 | 23 | x = Conv2D(64, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 24 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(preprocessed) 25 | x = BatchNormalization()(x) 26 | x = Activation('relu')(x) 27 | 28 | x = Conv2D(64, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 29 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) 30 | x = BatchNormalization()(x) 31 | x = Activation('relu')(x) 32 | 33 | x = AveragePooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x) 34 | 35 | x = Conv2D(96, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 36 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) 37 | x = BatchNormalization()(x) 38 | x = Activation('relu')(x) 39 | 40 | x = Conv2D(64, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 41 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) 42 | x = BatchNormalization()(x) 43 | x = Activation('relu')(x) 44 | 45 | x = Conv2D(96, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 46 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) 47 | x = BatchNormalization()(x) 48 | x = Activation('relu')(x) 49 | 50 | x = AveragePooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x) 51 | 52 | x = Conv2D(128, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 53 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) 54 | x = BatchNormalization()(x) 55 | x = Activation('relu')(x) 56 | 57 | x = Conv2D(96, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 58 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) 59 | x = BatchNormalization()(x) 60 | x = Activation('relu')(x) 61 | 62 | x = Conv2D(128, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 63 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) 64 | x = BatchNormalization()(x) 65 | x = Activation('relu')(x) 66 | 67 | x = AveragePooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x) 68 | 69 | x = Conv2D(256, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 70 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) 71 | x = BatchNormalization()(x) 72 | x = Activation('relu')(x) 73 | 74 | x = Conv2D(192, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 75 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) 76 | x = BatchNormalization()(x) 77 | x = Activation('relu')(x) 78 | 79 | x = Conv2D(256, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 80 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) 81 | x = BatchNormalization()(x) 82 | x = Activation('relu')(x) 83 | 84 | x = AveragePooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x) 85 | 86 | x = Conv2D(448, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 87 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) 88 | x = BatchNormalization()(x) 89 | x = Activation('relu')(x) 90 | 91 | x = Conv2D(256, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 92 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) 93 | x = BatchNormalization()(x) 94 | x = Activation('relu')(x) 95 | 96 | x = Conv2D(448, (3, 3), padding='same', strides=(1, 1), kernel_initializer='he_normal', use_bias=False, 97 | kernel_regularizer=l2(reg), bias_regularizer=l2(reg))(x) 98 | x = BatchNormalization()(x) 99 | x = Activation('relu')(x) 100 | 101 | return build_FC(input_image,x,settings) -------------------------------------------------------------------------------- /utils/korean_manager.py: -------------------------------------------------------------------------------- 1 | import json 2 | import numpy as np 3 | 4 | CHOSUNG_LIST = ['ㄱ', 'ㄲ', 'ㄴ', 'ㄷ', 'ㄸ', 'ㄹ', 'ㅁ', 'ㅂ', 'ㅃ', 'ㅅ', 'ㅆ', 'ㅇ', 'ㅈ', 'ㅉ', 'ㅊ', 'ㅋ', 'ㅌ', 'ㅍ', 'ㅎ', ' '] 5 | # 중성 리스트. 00 ~ 20 6 | JUNGSUNG_LIST = ['ㅏ', 'ㅐ', 'ㅑ', 'ㅒ', 'ㅓ', 'ㅔ', 'ㅕ', 'ㅖ', 'ㅗ', 'ㅘ', 'ㅙ', 'ㅚ', 'ㅛ', 'ㅜ', 'ㅝ', 'ㅞ', 'ㅟ', 'ㅠ', 'ㅡ', 'ㅢ', 'ㅣ', ' '] 7 | # 종성 리스트. 00 ~ 27 + 1(1개 없음) 8 | JONGSUNG_LIST = [' ', 'ㄱ', 'ㄲ', 'ㄳ', 'ㄴ', 'ㄵ', 'ㄶ', 'ㄷ', 'ㄹ', 'ㄺ', 'ㄻ', 'ㄼ', 'ㄽ', 'ㄾ', 'ㄿ', 'ㅀ', 'ㅁ', 'ㅂ', 'ㅄ', 'ㅅ', 'ㅆ', 'ㅇ', 'ㅈ', 'ㅊ', 'ㅋ', 'ㅌ', 'ㅍ', 'ㅎ'] 9 | 10 | def load_charset(charset='kr',charset_path='files/cjk.json'): 11 | #Load charset(list of characters): charset[index] = korean 12 | #Charset referenced from zi2zi: kr, jp, gbk2312, gbk2312_t, gbk 13 | 14 | with open(charset_path) as json_file: 15 | data = json.load(json_file) 16 | charset=data[charset] 17 | return charset 18 | def inverse_charset(charset='kr',charset_path='files/cjk.json'): 19 | #Load the inverse of the charset: inverse[korean] = index 20 | charset=load_charset(charset=charset,charset_path=charset_path) 21 | inverse={} 22 | for x in range(len(charset)): 23 | inverse[charset[x]]=x 24 | return inverse 25 | 26 | def index_to_korean(l): 27 | cho,jung,jong=l 28 | characterValue = ( (cho * 21) + jung) * 28 + jong + 0xAC00 29 | return chr(characterValue) 30 | 31 | def korean_numpy(words): 32 | charset=inverse_charset() 33 | arr=[] 34 | for w in words: 35 | ch1,ch2,ch3=korean_split(w) 36 | arr.append(charset[w]) 37 | return np.array(arr) 38 | 39 | def korean_split(w): 40 | ch1 = (ord(w) - ord('가'))//588 41 | ch2 = ((ord(w) - ord('가')) - (588*ch1)) // 28 42 | ch3 = (ord(w) - ord('가')) - (588*ch1) - 28*ch2 43 | 44 | #ㄱ,ㄴ,ㄷ,... 처리 45 | if ch1==-54: 46 | ch1,ch2,ch3=19,22,ord(w) - ord('ㄱ')+1 47 | return ch1,ch2,ch3 48 | 49 | def korean_split_numpy(words,to_text=False): 50 | # 한글 글자의 np array를 입력받아 초성, 중성, 종성을 각각의 array로 내보내는 함수 51 | cho,jung,jong=[],[],[] 52 | for w in words: 53 | ch1,ch2,ch3=korean_split(w) 54 | 55 | if to_text: 56 | cho.append(CHOSUNG_LIST[ch1]) 57 | jung.append(JUNGSUNG_LIST[ch2]) 58 | jong.append(JONGSUNG_LIST[ch3]) 59 | else: 60 | cho.append(ch1) 61 | jung.append(ch2) 62 | jong.append(ch3) 63 | 64 | return np.array(cho),np.array(jung),np.array(jong) -------------------------------------------------------------------------------- /utils/model_architectures.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import utils.korean_manager as korean_manager 4 | import utils.CustomLayers as CustomLayers 5 | from utils.model_components import build_FC,PreprocessingPipeline 6 | 7 | def VGG16(settings): 8 | if settings['direct_map']: 9 | input_channels=8 10 | else: 11 | input_channels=1 12 | VGG_net = tf.keras.applications.VGG16(input_shape=(settings['input_shape'],settings['input_shape'],input_channels), 13 | include_top=False,weights=None) 14 | 15 | input_image=tf.keras.layers.Input(shape=(settings['input_shape'],settings['input_shape']),name='input_image') 16 | preprocessed=tf.keras.layers.Reshape((settings['input_shape'],settings['input_shape'],1))(input_image) 17 | preprocessed=PreprocessingPipeline(settings['direct_map'],settings['data_augmentation'])(preprocessed) 18 | 19 | feature=VGG_net(preprocessed) 20 | 21 | return build_FC(input_image,x,settings) 22 | 23 | def EfficientCNN(settings): 24 | def fire_block(x,channels,stride=1): 25 | fire=tf.keras.layers.Conv2D(channels//8,kernel_size=1,padding='same')(x) 26 | firel=tf.keras.layers.BatchNormalization()(fire) 27 | fire=tf.keras.layers.LeakyReLU()(fire) 28 | 29 | fire1=tf.keras.layers.Conv2D(channels//2,kernel_size=3,padding='same',strides=(stride,stride))(fire) 30 | fire1=tf.keras.layers.BatchNormalization()(fire1) 31 | fire1=tf.keras.layers.LeakyReLU()(fire1) 32 | 33 | fire2=tf.keras.layers.Conv2D(channels//2,kernel_size=3,padding='same',strides=(stride,stride))(fire) 34 | fire2=tf.keras.layers.BatchNormalization()(fire2) 35 | fire2=tf.keras.layers.LeakyReLU()(fire2) 36 | 37 | fire=tf.keras.layers.concatenate([fire1,fire2]) 38 | return fire 39 | 40 | input_image=tf.keras.layers.Input(shape=(settings['input_shape'],settings['input_shape']),name='input_image') 41 | preprocessed=tf.keras.layers.Reshape((settings['input_shape'],settings['input_shape'],1))(input_image) 42 | preprocessed=PreprocessingPipeline(settings['direct_map'],settings['data_augmentation'])(preprocessed) 43 | 44 | conv1=tf.keras.layers.Conv2D(64,kernel_size=3,padding='same')(preprocessed) 45 | conv1=tf.keras.layers.BatchNormalization()(conv1) 46 | conv1=tf.keras.layers.LeakyReLU()(conv1) 47 | 48 | fire1=fire_block(conv1,64,2) 49 | fire2=fire_block(fire1,128) 50 | fire3=fire_block(fire2,128) 51 | fire4=fire_block(fire3,128,2) 52 | fire5=fire_block(fire4,256) 53 | fire6=fire_block(fire5,256) 54 | fire7=fire_block(fire6,256,2) 55 | fire8=fire_block(fire7,512) 56 | fire9=fire_block(fire8,512) 57 | fire10=fire_block(fire9,512) 58 | fire11=fire_block(fire10,512) 59 | 60 | return build_FC(input_image,fire11,settings) 61 | 62 | def InceptionResnetV2(settings): 63 | if settings['direct_map']: 64 | input_channels=8 65 | else: 66 | input_channels=1 67 | InceptionResnet = tf.keras.applications.InceptionResNetV2(input_shape=(settings['input_shape'],settings['input_shape'],input_channels), 68 | include_top=False,weights=None) 69 | 70 | input_image=tf.keras.layers.Input(shape=(settings['input_shape'],settings['input_shape']),name='input_image') 71 | preprocessed=tf.keras.layers.Reshape((settings['input_shape'],settings['input_shape'],1))(input_image) 72 | preprocessed=PreprocessingPipeline(settings['direct_map'],settings['data_augmentation'])(preprocessed) 73 | 74 | feature=InceptionResnet(preprocessed) 75 | 76 | return build_FC(input_image,x,settings) 77 | 78 | def MobilenetV3(settings): 79 | if settings['direct_map']: 80 | input_channels=8 81 | else: 82 | input_channels=1 83 | Mobilenet = tf.keras.applications.MobileNetV3Small(input_shape=(settings['input_shape'],settings['input_shape'],input_channels), 84 | include_top=False, weights=None) 85 | 86 | input_image=tf.keras.layers.Input(shape=(settings['input_shape'],settings['input_shape']),name='input_image') 87 | preprocessed=tf.keras.layers.Reshape((settings['input_shape'],settings['input_shape'],1))(input_image) 88 | preprocessed=PreprocessingPipeline(settings['direct_map'],settings['data_augmentation'])(preprocessed) 89 | 90 | feature=Mobilenet(preprocessed) 91 | 92 | return build_FC(input_image,x,settings) 93 | 94 | def AFL_Model(settings): 95 | if settings['direct_map']: 96 | input_channels=8 97 | else: 98 | input_channels=1 99 | input_image=tf.keras.layers.Input(shape=(settings['input_shape'],settings['input_shape']),name='input_image') 100 | preprocessed=tf.keras.layers.Reshape((settings['input_shape'],settings['input_shape'],1))(input_image) 101 | preprocessed=PreprocessingPipeline(settings['direct_map'],settings['data_augmentation'])(preprocessed) 102 | 103 | def conv_block(channels,kernel_size=3,strides=1,bn=True): 104 | m=tf.keras.models.Sequential([ 105 | tf.keras.layers.Conv2D(channels,kernel_size,strides=strides,padding='same'), 106 | tf.keras.layers.BatchNormalization(), 107 | tf.keras.layers.LeakyReLU() 108 | ]) 109 | return m 110 | 111 | x=conv_block(96)(preprocessed) 112 | x=conv_block(96,strides=2)(x) 113 | x=conv_block(128)(x) 114 | x=conv_block(128,strides=2)(x) 115 | x=conv_block(160)(x) 116 | x=conv_block(160,strides=2)(x) 117 | x=conv_block(256)(x) 118 | x=conv_block(256,strides=2)(x) 119 | x=conv_block(256)(x) 120 | 121 | return build_FC(input_image,x,settings) -------------------------------------------------------------------------------- /utils/model_components.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | from tensorflow.keras.models import Model 4 | import utils.korean_manager as korean_manager 5 | import utils.CustomLayers as CustomLayers 6 | 7 | def build_FC(input_image,x,settings): 8 | if settings['iterative_refinement']==True: 9 | preds=build_ir_split(input_image,x,settings) 10 | return Model(inputs=input_image,outputs=preds) 11 | 12 | if settings['split_components']: 13 | model=build_FC_split(input_image,x,settings) 14 | return model 15 | else: 16 | x=build_FC_regular(x) 17 | return Model(inputs=input_image,outputs=x) 18 | 19 | def build_FC_split(input_image,x,settings): 20 | if settings['fc_link']=='': 21 | x=tf.keras.layers.Flatten()(x) 22 | elif settings['fc_link']=='GAP': 23 | x=tf.keras.layers.GlobalAveragePooling2D()(x) 24 | elif settings['fc_link']=='GWAP': 25 | x=CustomLayers.GlobalWeightedAveragePooling()(x) 26 | 27 | dense=tf.keras.layers.Dropout(0.5)(x) 28 | dense=tf.keras.layers.Dense(2048)(dense) 29 | dense=tf.keras.layers.BatchNormalization()(dense) 30 | dense=tf.keras.layers.LeakyReLU()(dense) 31 | 32 | CHO=tf.keras.layers.Dense(len(korean_manager.CHOSUNG_LIST),activation='softmax',name='CHOSUNG')(dense) 33 | JUNG=tf.keras.layers.Dense(len(korean_manager.JUNGSUNG_LIST),activation='softmax',name='JUNGSUNG')(dense) 34 | JONG=tf.keras.layers.Dense(len(korean_manager.JONGSUNG_LIST),activation='softmax',name='JONGSUNG')(dense) 35 | #Define discriminator 36 | if settings['adversarial_learning']: 37 | disc=tf.keras.layers.Dense(512,name='disc_start')(x) 38 | disc=tf.keras.layers.LeakyReLU()(disc) 39 | disc=tf.keras.layers.Dropout(0.5)(disc) 40 | disc=tf.keras.layers.Dense(1,activation='sigmoid',name='DISC')(disc) 41 | 42 | return Model(inputs=input_image,outputs=[CHO,JUNG,JONG,disc]) 43 | 44 | return Model(inputs=input_image,outputs=[CHO,JUNG,JONG]) 45 | 46 | def build_ir_split(input_image,x,settings): 47 | units=512 48 | _,width,height,channels=x.shape 49 | x=tf.keras.layers.Reshape((width*height,channels))(x) 50 | #Define attention + GRU for each component 51 | CHO_RNN=CustomLayers.RNN_Decoder(units,len(korean_manager.CHOSUNG_LIST)) 52 | JUNG_RNN=CustomLayers.RNN_Decoder(units,len(korean_manager.JUNGSUNG_LIST)) 53 | JONG_RNN=CustomLayers.RNN_Decoder(units,len(korean_manager.JONGSUNG_LIST)) 54 | 55 | #Build RNN by looping the decoder t times 56 | zero_list=tf.tile(tf.zeros_like(input_image)[:,0:1,1],tf.constant([1,512])) 57 | hidden_CHO, hidden_JUNG, hidden_JONG = zero_list,zero_list,zero_list 58 | pred_list=[] 59 | 60 | for timestep in range(settings['refinement_t']): 61 | pred_CHO, hidden_CHO,_ = CHO_RNN(x, hidden_CHO) 62 | pred_JUNG, hidden_CHO,_ = JUNG_RNN(x, hidden_JUNG) 63 | pred_JONG, hidden_CHO,_ = JONG_RNN(x, hidden_JONG) 64 | 65 | if not timestep==settings['refinement_t']-1: 66 | index=str(timestep) 67 | else: 68 | index='' 69 | #Rename layers by mapping tensors into layers 70 | pred_CHO = tf.keras.layers.Lambda(lambda val:val,name='CHOSUNG'+index)(pred_CHO) 71 | pred_JUNG = tf.keras.layers.Lambda(lambda val:val,name='JUNGSUNG'+index)(pred_JUNG) 72 | pred_JONG = tf.keras.layers.Lambda(lambda val:val,name='JONGSUNG'+index)(pred_JONG) 73 | pred_list+=[pred_CHO,pred_JUNG,pred_JONG] 74 | 75 | return pred_list 76 | 77 | def build_FC_regular(x): 78 | x=tf.keras.layers.Flatten()(x) 79 | x=tf.keras.layers.Dense(1024)(x) 80 | x=tf.keras.layers.Dense(len(korean_manager.load_charset()),activation='softmax',name='output')(x) 81 | return x 82 | 83 | def PreprocessingPipeline(direct_map,data_augmentation): 84 | preprocessing=tf.keras.models.Sequential() 85 | 86 | #[0, 255] to [0, 1] with black white reversed 87 | preprocessing.add(tf.keras.layers.experimental.preprocessing.Rescaling(scale=-1/255,offset=1)) 88 | #Whether to perform data augmentation 89 | if data_augmentation==True: 90 | preprocessing.add(DataAugmentation()) 91 | #DirectMap normalization 92 | if direct_map==True: 93 | preprocessing.add(DirectMapGeneration()) 94 | return preprocessing 95 | 96 | def DataAugmentation(): 97 | augment = tf.keras.Sequential([ 98 | tf.keras.layers.experimental.preprocessing.RandomZoom( height_factor=(-0.2, 0.1),width_factor=(-0.2, 0.1),fill_mode='constant'), 99 | tf.keras.layers.experimental.preprocessing.RandomRotation(0.1,fill_mode='constant'), 100 | tf.keras.layers.experimental.preprocessing.RandomTranslation(0.1,0.1,fill_mode='constant') 101 | 102 | ]) 103 | return augment 104 | 105 | def DirectMapGeneration(): 106 | #Generate sobel filter for 8 direction maps 107 | sobel_filters=[ 108 | [[-1,0,1], 109 | [-2,0,2], 110 | [-1,0,1]], 111 | 112 | [[1,0,-1], 113 | [2,0,-2], 114 | [1,0,-1]], 115 | 116 | [[1,2,1], 117 | [0,0,0], 118 | [-1,-2,-1]], 119 | 120 | [[-1,-2,-1], 121 | [0,0,0], 122 | [1,2,1]], 123 | 124 | [[0,1,2], 125 | [-1,0,1], 126 | [-2,-1,0]], 127 | 128 | [[0,-1,-2], 129 | [1,0,-1], 130 | [2,1,0]], 131 | 132 | [[-2,-1,0], 133 | [-1,0,1], 134 | [0,1,2]], 135 | 136 | [[2,1,0], 137 | [1,0,-1], 138 | [0,-1,-2]]] 139 | sobel_filters=np.array(sobel_filters).astype(np.float).reshape(8,3,3,1) 140 | sobel_filters=np.moveaxis(sobel_filters, 0, -1) 141 | 142 | DirectMap = tf.keras.models.Sequential() 143 | DirectMap.add(tf.keras.layers.Conv2D(8, (3,3), padding='same',input_shape=(None, None, 1),use_bias=False)) 144 | DirectMap.set_weights([sobel_filters]) 145 | return DirectMap -------------------------------------------------------------------------------- /utils/predict_char.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import utils.korean_manager as korean_manager 3 | 4 | def predict_complete(model,image,n=1): 5 | #Predict the top-n classes of the image 6 | #Returns top n characters that maximize the probability 7 | charset=korean_manager.load_charset() 8 | if image.shape==(256,256): 9 | image=image.reshape((1,256,256)) 10 | pred_class=model.predict(image) 11 | pred_class=np.argsort(pred_class,axis=1)[:,-n:] 12 | pred_hangeul=[] 13 | for idx in range(image.shape[0]): 14 | pred_hangeul.append([]) 15 | for char in pred_class[idx]: 16 | pred_hangeul[-1].append(charset[char]) 17 | return pred_hangeul 18 | 19 | def split_topn(cho_pred,jung_pred,jong_pred,n): 20 | k=int(n**(1/3))+2 21 | cho_idx,jung_idx,jong_idx=np.argsort(cho_pred,axis=1)[:,-k:],np.argsort(jung_pred,axis=1)[:,-k:],np.argsort(jong_pred,axis=1)[:,-k:] 22 | cho_pred,jung_pred,jong_pred=np.sort(cho_pred,axis=1)[:,-k:],np.sort(jung_pred,axis=1)[:,-k:],np.sort(jong_pred,axis=1)[:,-k:] 23 | #Convert indicies to korean character 24 | pred_hangeul=[] 25 | for idx in range(cho_pred.shape[0]): 26 | pred_hangeul.append([]) 27 | 28 | cho_prob,jung_prob,jong_prob=cho_pred[idx],jung_pred[idx].reshape(-1,1),jong_pred[idx].reshape(-1,1) 29 | 30 | mult=((cho_prob*jung_prob).flatten()*jong_prob).flatten().argsort()[-5:][::-1] 31 | for max_idx in mult: 32 | pred_hangeul[-1].append(korean_manager.index_to_korean((cho_idx[idx][max_idx%k],jung_idx[idx][(max_idx%(k*k))//k]\ 33 | ,jong_idx[idx][max_idx//(k*k)]))) 34 | 35 | return pred_hangeul 36 | 37 | def predict_ir(model,image,n=1, t=4): 38 | def find_target_layer(model): 39 | # attempt to find the final convolutional layer in the network 40 | # by looping over the layers of the network in reverse order 41 | for layer in reversed(model.layers): 42 | # check to see if the layer has a 4D output 43 | if len(layer.output_shape) == 4: 44 | return layer.name 45 | # otherwise, we could not find a 4D layer so the GradCAM 46 | # algorithm cannot be applied 47 | raise ValueError("Could not find 4D layer. Cannot apply GradCAM.") 48 | return 0 49 | 50 | def predict_split(model,image,n=1): 51 | #Predict the top-n classes of the image 52 | #k: top classes for each component to generate 53 | #Returns top n characters that maximize pred(chosung)*pred(jungsung)*pred(jongsung) 54 | 55 | if image.shape==(256,256): 56 | image=image.reshape((1,256,256)) 57 | #Predict top n classes 58 | 59 | prediction_list=model.predict(image) 60 | prediction_dict = {name: pred for name, pred in zip(model.output_names, prediction_list)} 61 | #cho_pred,jung_pred,jong_pred=prediction_dict['CHOSUNG'],prediction_dict['JUNGSUNG'],prediction_dict['JONGSUNG'] 62 | cho_pred,jung_pred,jong_pred=prediction_list[0],prediction_list[1],prediction_list[2] 63 | return split_topn(cho_pred,jung_pred,jong_pred,n) --------------------------------------------------------------------------------