├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── assets └── overlay.png ├── build.rs ├── config └── log4rs.yaml ├── resources ├── comictextdetector_blk.pt.onnx ├── fonts │ ├── NotoSansJP-Regular.ttf │ └── OFL.txt ├── icon-256.ico └── icon-256.png └── src ├── action.rs ├── database ├── history_data.rs ├── kanji_statistic.rs ├── mod.rs └── table.rs ├── detect ├── comictextdetector.rs ├── mod.rs └── session_builder.rs ├── jpn ├── dict.rs ├── kanji.json ├── kanji.rs └── mod.rs ├── lib.rs ├── main.rs ├── ocr ├── manga_ocr.rs └── mod.rs ├── translation ├── google.rs └── mod.rs └── ui ├── app.rs ├── background_rect.rs ├── event.rs ├── kanji_history_ui.rs ├── kanji_statistic_ui.rs ├── mod.rs ├── mouse_hover.rs ├── screenshot_result_ui.rs ├── settings.rs └── shutdown.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /output 3 | /input 4 | /log 5 | /.idea/ 6 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "manga_overlay" 3 | version = "1.1.0" 4 | edition = "2024" 5 | build = "build.rs" 6 | 7 | [dependencies] 8 | eframe = { version = "0.31.0", default-features = false, features = [ 9 | "default_fonts", # Embed the default egui fonts. 10 | "glow", # Use the glow rendering backend. Alternative: "wgpu". 11 | "persistence", # Enable restoring app state when restarting the app. # To support Linux (and CI) 12 | ] } 13 | egui = "0.31.0" 14 | egui_extras = "0.31.0" 15 | enigo = "0.3.0" 16 | log = "0.4" 17 | # You only need serde if you want app persistence: 18 | serde = { version = "1", features = ["derive", "rc"] } 19 | 20 | 21 | serde_json = "1.0" 22 | rusty-tesseract = "1.1.9" 23 | screenshots = "0.8.10" 24 | anyhow = "1.0.80" 25 | jmdict = "2.0.0" 26 | tokio = { version = "1", features = ["full"] } 27 | tokio-util = { version = "0.7.13", features = ["rt"] } 28 | futures = "0.3.28" 29 | itertools = "0.12.1" 30 | multimap = "0.10.0" 31 | serde_with = "3.3.0" 32 | strum = { version = "0.26.1", features = ["derive"] } 33 | scraper = "0.19.0" 34 | reqwest = "0.11.23" 35 | ort = { version = "2.0.0-rc.9", features = ["cuda"] } 36 | ndarray = "0.16.1" 37 | image = "0.25.5" 38 | imageproc = "0.25.0" 39 | log4rs = "1.3.0" 40 | open = "5.3.0" 41 | rusqlite = { version = "0.32.0", features = ["bundled"] } 42 | hf-hub = "0.4.2" 43 | candle-transformers = "0.8.4" 44 | 45 | 46 | 47 | [profile.release] 48 | opt-level = 3 # fast and small wasm 49 | 50 | # Optimize all dependencies even in debug builds: 51 | [profile.dev.package."*"] 52 | opt-level = 2 53 | 54 | 55 | 56 | [dev-dependencies] 57 | serial_test = "3.2.0" 58 | 59 | [build-dependencies] 60 | winres = "0.1.12" 61 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Manga Overlay 2 | 3 | Desktop Overlay for Japanese manga. The primary focus is learning japanese by making the search for kanji meanings 4 | faster to enable reading manga. 5 | 6 | Currently only Windows is supported. 7 | 8 | ## Setup 9 | - Install [Rust](https://www.rust-lang.org/tools/install) 10 | - Clone the repo with [Git](https://git-scm.com/downloads/win) 11 | - Optional install Cuda for faster OCR 12 | - Build a exe with "cargo build --release" or run with "cargo run" 13 | 14 | ## Usage 15 | 16 | Select an area by dragging the background of the overlay. The app detects japanese text in the selected area and shows 17 | the 18 | result when hovering the blue rectangles. Scrolling in the blue rect shows the meanings of the detected kanji. 19 | With a left click on the rect the text gets send to Google for translation and the result is cached in a sqlite db. 20 | A right click keeps the info textbox open. 21 | 22 | With "mouse passthrough" turned on the background becomes click through. This enables continues detection of japanese 23 | text 24 | in combination with the auto restart feature. 25 | While hovering over a detected rect the ocr is paused. 26 | 27 | A history of detected text can be displayed by enabling the "Show History" checkbox.\ 28 | With "Show Statistics" a basic overview of often looked at kanji is displayed. 29 | 30 | ![overlay.png](assets/overlay.png) 31 | 32 | ## License 33 | 34 | This project is licensed under the GPL-3.0 License - see the [LICENSE](LICENSE) file for details. 35 | 36 | ## Acknowledgments 37 | 38 | This project was done with the usage of: 39 | 40 | - [egui](https://github.com/emilk/egui) gui overlay 41 | - [kanji-data](https://github.com/davidluzgouveia/kanji-data) kanji meaning dataset 42 | - [comic-text-detector](https://github.com/dmMaze/comic-text-detector) trimmed model for textbox detection 43 | - [manga-ocr](https://github.com/kha-white/manga-ocr) model/python scripts for text detection 44 | - [koharu](https://github.com/mayocream/koharu) onnx models and scripts for text detection 45 | -------------------------------------------------------------------------------- /assets/overlay.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Icekey/manga-overlay/7929070b94decda14fc7a8b69e72000917b6772b/assets/overlay.png -------------------------------------------------------------------------------- /build.rs: -------------------------------------------------------------------------------- 1 | extern crate winres; 2 | 3 | fn main() { 4 | if cfg!(target_os = "windows") { 5 | let mut res = winres::WindowsResource::new(); 6 | res.set_icon("resources/icon-256.ico"); 7 | res.compile().unwrap(); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /config/log4rs.yaml: -------------------------------------------------------------------------------- 1 | refresh_rate: 30 seconds 2 | appenders: 3 | stdout: 4 | kind: console 5 | encoder: 6 | pattern: "{h({l})} {d(%Y-%m-%d %H:%M:%S)} {f}:{L} - {m}{n}" 7 | requests: 8 | kind: file 9 | path: "log/manga_overlay.log" 10 | encoder: 11 | pattern: "{h({l})} {d(%Y-%m-%d %H:%M:%S)} {f}:{L} - {m}{n}" 12 | root: 13 | level: info 14 | appenders: 15 | - stdout 16 | - requests -------------------------------------------------------------------------------- /resources/comictextdetector_blk.pt.onnx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Icekey/manga-overlay/7929070b94decda14fc7a8b69e72000917b6772b/resources/comictextdetector_blk.pt.onnx -------------------------------------------------------------------------------- /resources/fonts/NotoSansJP-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Icekey/manga-overlay/7929070b94decda14fc7a8b69e72000917b6772b/resources/fonts/NotoSansJP-Regular.ttf -------------------------------------------------------------------------------- /resources/fonts/OFL.txt: -------------------------------------------------------------------------------- 1 | Copyright 2014-2021 Adobe (http://www.adobe.com/), with Reserved Font Name 'Source' 2 | 3 | This Font Software is licensed under the SIL Open Font License, Version 1.1. 4 | This license is copied below, and is also available with a FAQ at: 5 | https://openfontlicense.org 6 | 7 | 8 | ----------------------------------------------------------- 9 | SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 10 | ----------------------------------------------------------- 11 | 12 | PREAMBLE 13 | The goals of the Open Font License (OFL) are to stimulate worldwide 14 | development of collaborative font projects, to support the font creation 15 | efforts of academic and linguistic communities, and to provide a free and 16 | open framework in which fonts may be shared and improved in partnership 17 | with others. 18 | 19 | The OFL allows the licensed fonts to be used, studied, modified and 20 | redistributed freely as long as they are not sold by themselves. The 21 | fonts, including any derivative works, can be bundled, embedded, 22 | redistributed and/or sold with any software provided that any reserved 23 | names are not used by derivative works. The fonts and derivatives, 24 | however, cannot be released under any other type of license. The 25 | requirement for fonts to remain under this license does not apply 26 | to any document created using the fonts or their derivatives. 27 | 28 | DEFINITIONS 29 | "Font Software" refers to the set of files released by the Copyright 30 | Holder(s) under this license and clearly marked as such. This may 31 | include source files, build scripts and documentation. 32 | 33 | "Reserved Font Name" refers to any names specified as such after the 34 | copyright statement(s). 35 | 36 | "Original Version" refers to the collection of Font Software components as 37 | distributed by the Copyright Holder(s). 38 | 39 | "Modified Version" refers to any derivative made by adding to, deleting, 40 | or substituting -- in part or in whole -- any of the components of the 41 | Original Version, by changing formats or by porting the Font Software to a 42 | new environment. 43 | 44 | "Author" refers to any designer, engineer, programmer, technical 45 | writer or other person who contributed to the Font Software. 46 | 47 | PERMISSION & CONDITIONS 48 | Permission is hereby granted, free of charge, to any person obtaining 49 | a copy of the Font Software, to use, study, copy, merge, embed, modify, 50 | redistribute, and sell modified and unmodified copies of the Font 51 | Software, subject to the following conditions: 52 | 53 | 1) Neither the Font Software nor any of its individual components, 54 | in Original or Modified Versions, may be sold by itself. 55 | 56 | 2) Original or Modified Versions of the Font Software may be bundled, 57 | redistributed and/or sold with any software, provided that each copy 58 | contains the above copyright notice and this license. These can be 59 | included either as stand-alone text files, human-readable headers or 60 | in the appropriate machine-readable metadata fields within text or 61 | binary files as long as those fields can be easily viewed by the user. 62 | 63 | 3) No Modified Version of the Font Software may use the Reserved Font 64 | Name(s) unless explicit written permission is granted by the corresponding 65 | Copyright Holder. This restriction only applies to the primary font name as 66 | presented to the users. 67 | 68 | 4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font 69 | Software shall not be used to promote, endorse or advertise any 70 | Modified Version, except to acknowledge the contribution(s) of the 71 | Copyright Holder(s) and the Author(s) or with their explicit written 72 | permission. 73 | 74 | 5) The Font Software, modified or unmodified, in part or in whole, 75 | must be distributed entirely under this license, and must not be 76 | distributed under any other license. The requirement for fonts to 77 | remain under this license does not apply to any document created 78 | using the Font Software. 79 | 80 | TERMINATION 81 | This license becomes null and void if any of the above conditions are 82 | not met. 83 | 84 | DISCLAIMER 85 | THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 86 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF 87 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 88 | OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE 89 | COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 90 | INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL 91 | DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 92 | FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM 93 | OTHER DEALINGS IN THE FONT SOFTWARE. 94 | -------------------------------------------------------------------------------- /resources/icon-256.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Icekey/manga-overlay/7929070b94decda14fc7a8b69e72000917b6772b/resources/icon-256.ico -------------------------------------------------------------------------------- /resources/icon-256.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Icekey/manga-overlay/7929070b94decda14fc7a8b69e72000917b6772b/resources/icon-256.png -------------------------------------------------------------------------------- /src/action.rs: -------------------------------------------------------------------------------- 1 | use crate::database::{HistoryData, KanjiStatistic}; 2 | use futures::future::join_all; 3 | use image::DynamicImage; 4 | use imageproc::rect::Rect; 5 | use itertools::Itertools; 6 | use log::info; 7 | use open::that; 8 | use ::serde::{Deserialize, Serialize}; 9 | 10 | use crate::detect::comictextdetector::{combine_overlapping_rects, Boxes, DETECT_STATE}; 11 | use crate::jpn::{dict, get_jpn_data, JpnData}; 12 | use crate::ocr::OcrBackend; 13 | use crate::translation::google::translate; 14 | use crate::{database, detect}; 15 | 16 | pub fn open_workdir() { 17 | let current_dir = std::env::current_dir().expect("Failed to get current_dir"); 18 | that(current_dir).expect("Failed to open current_dir"); 19 | } 20 | 21 | #[derive(Serialize, Deserialize, PartialEq, Debug, Default)] 22 | pub struct ScreenshotParameter { 23 | pub x: i32, 24 | pub y: i32, 25 | pub width: u32, 26 | pub height: u32, 27 | pub detect_boxes: bool, 28 | pub full_capture_ocr: bool, 29 | pub backends: Vec, 30 | pub threshold: f32, 31 | } 32 | 33 | pub async fn run_ocr( 34 | parameter: ScreenshotParameter, 35 | mut capture_image: DynamicImage, 36 | ) -> Result { 37 | let backends: Vec = parameter.backends; 38 | 39 | //Detect Boxes 40 | let all_boxes: Vec = if parameter.detect_boxes { 41 | DETECT_STATE 42 | .clone() 43 | .run_model(parameter.threshold, &mut capture_image) 44 | } else { 45 | vec![] 46 | }; 47 | 48 | let boxes = combine_overlapping_rects(all_boxes.clone()); 49 | 50 | //Run OCR on Boxes 51 | let mut rects: Vec = boxes.iter().map(|x| x.get_rect(&capture_image)).collect(); 52 | 53 | if parameter.full_capture_ocr { 54 | //Add full image rect 55 | rects.insert( 56 | 0, 57 | Rect::at(0, 0).of_size(capture_image.width(), capture_image.height()), 58 | ); 59 | } 60 | 61 | let image = capture_image.clone(); 62 | 63 | let cutout_results = run_ocr_on_cutout_images(&image, &backends, rects); 64 | 65 | let mut futures = vec![]; 66 | 67 | for cutout_result in cutout_results { 68 | futures.push(get_result_data(cutout_result.0, cutout_result.1)); 69 | } 70 | 71 | let ocr_results: Vec = join_all(futures).await.into_iter().collect(); 72 | 73 | for ocr_result in &ocr_results { 74 | //Store OCR 75 | database::store_ocr(&ocr_result.ocr).expect("Failed to store ocr"); 76 | 77 | for jpn_data in ocr_result.jpn.iter().flatten() { 78 | if jpn_data.has_kanji_data() { 79 | //Store Kanji statistic 80 | database::init_kanji_statistic(&jpn_data.get_kanji()) 81 | .expect("Failed to store kanji"); 82 | } 83 | } 84 | } 85 | 86 | //Draw Boxes 87 | let capture_image = capture_image.clone(); 88 | let mut debug_image = capture_image.clone(); 89 | detect::comictextdetector::draw_rects(&mut debug_image, &all_boxes); 90 | 91 | Ok(ScreenshotResult { 92 | capture_image: Some(capture_image), 93 | debug_image: Some(debug_image), 94 | ocr_results, 95 | }) 96 | } 97 | 98 | fn run_ocr_on_cutout_images( 99 | capture_image: &DynamicImage, 100 | backends: &[OcrBackend], 101 | rects: Vec, 102 | ) -> Vec<(String, Rect)> { 103 | let cutout_images: Vec = rects 104 | .iter() 105 | .map(|x| get_cutout_image(capture_image, x)) 106 | .filter(|x| x.width() != 0 && x.height() != 0) 107 | .collect(); 108 | 109 | OcrBackend::run_backends(&cutout_images, backends) 110 | .into_iter() 111 | .zip(rects) 112 | .collect() 113 | } 114 | 115 | async fn get_result_data(ocr: String, rect: Rect) -> ResultData { 116 | let jpn: Vec> = get_jpn_data(&ocr).await; 117 | 118 | let translation = match database::load_history_data(&ocr) { 119 | Ok(x) => x.translation.unwrap_or_default(), 120 | Err(_) => String::new(), 121 | }; 122 | 123 | ResultData { 124 | x: rect.left(), 125 | y: rect.top(), 126 | w: rect.width() as i32, 127 | h: rect.height() as i32, 128 | ocr, 129 | translation, 130 | jpn, 131 | } 132 | } 133 | 134 | fn get_cutout_image(capture_image: &DynamicImage, rect: &Rect) -> DynamicImage { 135 | capture_image.crop_imm( 136 | rect.left() as u32, 137 | rect.top() as u32, 138 | rect.width(), 139 | rect.height(), 140 | ) 141 | } 142 | 143 | #[derive(Deserialize, Serialize, Default, Clone, Debug)] 144 | #[serde(default)] 145 | pub struct ScreenshotResult { 146 | #[serde(skip)] 147 | pub capture_image: Option, 148 | #[serde(skip)] 149 | pub debug_image: Option, 150 | pub ocr_results: Vec, 151 | } 152 | 153 | #[derive(Deserialize, Serialize, Default, Clone)] 154 | #[serde(default)] 155 | pub struct ResultData { 156 | pub x: i32, 157 | pub y: i32, 158 | pub w: i32, 159 | pub h: i32, 160 | pub ocr: String, 161 | pub translation: String, 162 | pub jpn: Vec>, 163 | } 164 | 165 | impl std::fmt::Debug for ResultData { 166 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 167 | f.debug_struct("ResultData") 168 | .field("x", &self.x) 169 | .field("y", &self.y) 170 | .field("w", &self.w) 171 | .field("h", &self.h) 172 | .field("ocr", &self.ocr) 173 | .finish() 174 | } 175 | } 176 | 177 | impl ResultData { 178 | pub fn get_jpn_data_with_info_count(&self) -> usize { 179 | self.get_jpn_data_with_info().count() 180 | } 181 | 182 | pub fn get_jpn_data_with_info_by_index(&self, index: i32) -> Option<&JpnData> { 183 | let count = self.get_jpn_data_with_info_count() as i32; 184 | if count == 0 { 185 | return None; 186 | } 187 | self.get_jpn_data_with_info() 188 | .nth(index.rem_euclid(count) as usize) 189 | } 190 | 191 | fn get_jpn_data_with_info(&self) -> impl Iterator { 192 | self.jpn.iter().flatten().filter(|y| y.has_kanji_data()) 193 | } 194 | } 195 | 196 | pub async fn get_translation(input: &str) -> String { 197 | use std::time::Instant; 198 | let now = Instant::now(); 199 | 200 | info!("Start get_translation"); 201 | 202 | let input = input.lines().map(dict::remove_whitespace).join("\n"); 203 | 204 | let elapsed = now.elapsed(); 205 | info!("End get_translation elapsed: {elapsed:.2?}"); 206 | 207 | let translation = translate(&input) 208 | .await 209 | .map_err(|err| err.to_string()) 210 | .unwrap_or_else(|err_string| err_string) 211 | .trim() 212 | .to_string(); 213 | 214 | database::store_ocr_translation(&input, &translation).expect("Failed to store history data"); 215 | 216 | translation 217 | } 218 | 219 | pub fn load_history() -> Vec { 220 | database::load_full_history().unwrap_or_else(|err| { 221 | log::error!("Failed to load history: {err}"); 222 | vec![] 223 | }) 224 | } 225 | 226 | pub fn increment_kanji_statistic(kanji: &str) -> KanjiStatistic { 227 | database::increment_kanji_statistic(kanji).expect("Failed to increment kanji statistic") 228 | } 229 | 230 | pub(crate) fn load_statistic() -> Vec { 231 | database::load_statistic().unwrap_or_else(|err| { 232 | log::error!("Failed to load statistic: {err}"); 233 | vec![] 234 | }) 235 | } 236 | 237 | pub async fn get_kanji_jpn_data(kanji: &str) -> Option { 238 | let vec = get_jpn_data(kanji).await; 239 | vec.into_iter().flatten().next() 240 | } 241 | 242 | #[cfg(test)] 243 | mod tests { 244 | 245 | #[tokio::test(flavor = "multi_thread")] 246 | async fn test_name() { 247 | //load DynamicImage 248 | // let image = image::open("../input/input.jpg").expect("Failed to open image"); 249 | // let run_ocr = run_ocr( 250 | // ScreenshotParameter { 251 | // detect_boxes: true, 252 | // backends: vec![OcrBackend::MangaOcr], 253 | // ..ScreenshotParameter::default() 254 | // }, 255 | // image, 256 | // ) 257 | // .await; 258 | 259 | // dbg!(run_ocr); 260 | assert!(true); 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /src/database/history_data.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{Ok, Result}; 2 | use rusqlite::{params, Connection, Row}; 3 | use serde::Serialize; 4 | 5 | use super::table::create_table; 6 | 7 | #[derive(Debug, Default, PartialEq, Serialize, serde::Deserialize, Clone)] 8 | pub struct HistoryData { 9 | pub id: i32, 10 | pub created_at: String, 11 | pub updated_at: String, 12 | pub ocr: String, 13 | pub translation: Option, 14 | } 15 | 16 | fn open_connection() -> Result { 17 | create_table( 18 | "CREATE TABLE IF NOT EXISTS history ( 19 | id INTEGER PRIMARY KEY, 20 | created_at TEXT NOT NULL DEFAULT current_timestamp, 21 | updated_at TEXT NOT NULL DEFAULT current_timestamp, 22 | ocr TEXT UNIQUE NOT NULL, 23 | translation TEXT 24 | )", 25 | ) 26 | } 27 | 28 | pub fn store_ocr(ocr: &str) -> Result<()> { 29 | let conn = open_connection()?; 30 | 31 | conn.execute( 32 | "INSERT INTO history (ocr) VALUES (?1) \ 33 | ON CONFLICT(ocr) DO NOTHING", 34 | params![ocr], 35 | )?; 36 | 37 | Ok(()) 38 | } 39 | 40 | pub fn store_ocr_translation(ocr: &str, translation: &str) -> Result<()> { 41 | let conn = open_connection()?; 42 | 43 | conn.execute( 44 | "INSERT INTO history (ocr, translation) VALUES (?1, ?2) \ 45 | ON CONFLICT(ocr) DO UPDATE SET translation = excluded.translation, updated_at = current_timestamp", 46 | params![ocr, translation], 47 | )?; 48 | 49 | Ok(()) 50 | } 51 | 52 | impl HistoryData { 53 | fn from_row(row: &Row<'_>) -> rusqlite::Result { 54 | let id: i32 = row.get(0)?; 55 | let created_at: String = row.get(1)?; 56 | let updated_at: String = row.get(2)?; 57 | let ocr: String = row.get(3)?; 58 | let translation: Option = row.get(4)?; 59 | 60 | rusqlite::Result::Ok(HistoryData { 61 | id, 62 | created_at, 63 | updated_at, 64 | ocr, 65 | translation, 66 | }) 67 | } 68 | } 69 | 70 | pub fn load_history_data(ocr: &str) -> Result { 71 | let conn = open_connection()?; 72 | 73 | let mut stmt = conn.prepare("SELECT * FROM history WHERE ocr = ?1")?; 74 | 75 | let history: HistoryData = stmt.query_row([ocr], HistoryData::from_row)?; 76 | 77 | Ok(history) 78 | } 79 | 80 | pub fn load_full_history() -> Result> { 81 | let conn = open_connection()?; 82 | 83 | let mut stmt = conn.prepare("SELECT * FROM history ORDER BY updated_at DESC, id DESC")?; 84 | 85 | let history: Vec = stmt 86 | .query_map([], HistoryData::from_row)? 87 | .collect::>>()?; 88 | 89 | Ok(history) 90 | } 91 | 92 | #[cfg(test)] 93 | mod tests { 94 | use serial_test::serial; 95 | 96 | use crate::database::table::drop_table; 97 | 98 | use super::*; 99 | 100 | #[test] 101 | #[serial] 102 | fn store_and_load_history() { 103 | drop_table("history").unwrap(); 104 | 105 | store_ocr("ocr1").unwrap(); 106 | store_ocr("ocr2").unwrap(); 107 | 108 | let vec = load_full_history().unwrap(); 109 | 110 | assert_eq!(&vec[0].ocr, "ocr2"); 111 | assert!(&vec[0].translation.is_none()); 112 | assert_eq!(&vec[1].ocr, "ocr1"); 113 | assert!(&vec[1].translation.is_none()); 114 | 115 | std::thread::sleep(std::time::Duration::from_secs(2)); 116 | 117 | store_ocr_translation("ocr1", "translation1").unwrap(); 118 | 119 | let vec = dbg!(load_full_history().unwrap()); 120 | 121 | assert_eq!(&vec[0].ocr, "ocr1"); 122 | assert_eq!(&vec[0].translation, &Some("translation1".to_string())); 123 | assert_eq!(&vec[1].ocr, "ocr2"); 124 | assert!(&vec[1].translation.is_none()); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/database/kanji_statistic.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{Ok, Result}; 2 | use rusqlite::{params, Connection, Row}; 3 | use serde::Serialize; 4 | 5 | use super::table::create_table; 6 | 7 | #[derive(Debug, Clone, Serialize, serde::Deserialize, PartialEq, Default)] 8 | pub struct KanjiStatistic { 9 | pub id: i32, 10 | pub created_at: String, 11 | pub updated_at: String, 12 | pub kanji: String, 13 | pub count: i32, 14 | } 15 | 16 | fn open_connection() -> Result { 17 | create_table( 18 | "CREATE TABLE IF NOT EXISTS statistic ( 19 | id INTEGER PRIMARY KEY, 20 | created_at TEXT NOT NULL DEFAULT current_timestamp, 21 | updated_at TEXT NOT NULL DEFAULT current_timestamp, 22 | kanji TEXT UNIQUE NOT NULL, 23 | count INTEGER NOT NULL DEFAULT 0 24 | )", 25 | ) 26 | } 27 | 28 | pub fn init_kanji_statistic(kanji: &str) -> Result { 29 | update_kanji_statistic( 30 | kanji, 31 | "INSERT INTO statistic (kanji) VALUES (?1) \ 32 | ON CONFLICT(kanji) DO NOTHING", 33 | ) 34 | } 35 | 36 | pub fn increment_kanji_statistic(kanji: &str) -> Result { 37 | update_kanji_statistic( 38 | kanji, 39 | "INSERT INTO statistic (kanji, count) VALUES (?1, 1) \ 40 | ON CONFLICT(kanji) DO UPDATE SET count = count + 1, updated_at = current_timestamp", 41 | ) 42 | } 43 | 44 | fn update_kanji_statistic(kanji: &str, query: &str) -> Result { 45 | let conn = open_connection()?; 46 | 47 | conn.execute(query, params![kanji])?; 48 | 49 | load_kanji_statistic(kanji) 50 | } 51 | 52 | pub fn load_kanji_statistic(kanji: &str) -> Result { 53 | let conn = open_connection()?; 54 | 55 | let mut stmt = conn.prepare("SELECT * FROM statistic WHERE kanji = ?1")?; 56 | 57 | let statistic: KanjiStatistic = stmt.query_row([kanji], KanjiStatistic::from_row)?; 58 | 59 | Ok(statistic) 60 | } 61 | 62 | impl KanjiStatistic { 63 | fn from_row(row: &Row<'_>) -> rusqlite::Result { 64 | let id: i32 = row.get(0)?; 65 | let created_at: String = row.get(1)?; 66 | let updated_at: String = row.get(2)?; 67 | let kanji: String = row.get(3)?; 68 | let count: i32 = row.get(4)?; 69 | 70 | rusqlite::Result::Ok(KanjiStatistic { 71 | id, 72 | created_at, 73 | updated_at, 74 | kanji, 75 | count, 76 | }) 77 | } 78 | } 79 | 80 | pub fn load_statistic() -> Result> { 81 | let conn = open_connection()?; 82 | 83 | let mut stmt = conn.prepare("SELECT * FROM statistic ORDER BY count DESC")?; 84 | 85 | let statistics: Vec = stmt 86 | .query_map([], KanjiStatistic::from_row)? 87 | .collect::>>()?; 88 | 89 | Ok(statistics) 90 | } 91 | 92 | #[cfg(test)] 93 | mod tests { 94 | use serial_test::serial; 95 | 96 | use crate::database::table::drop_table; 97 | 98 | use super::*; 99 | 100 | #[test] 101 | #[serial] 102 | fn store_and_load_statistic() { 103 | drop_table("statistic").unwrap(); 104 | 105 | init_kanji_statistic("Test1").unwrap(); 106 | increment_kanji_statistic("Test1").unwrap(); 107 | 108 | increment_kanji_statistic("Test2").unwrap(); 109 | increment_kanji_statistic("Test2").unwrap(); 110 | 111 | let result = load_statistic().unwrap(); 112 | 113 | assert_eq!(result.len(), 2); 114 | 115 | assert_eq!(result[0].kanji, "Test2"); 116 | assert_eq!(result[0].count, 2); 117 | assert_eq!(result[1].kanji, "Test1"); 118 | assert_eq!(result[1].count, 1); 119 | } 120 | 121 | #[test] 122 | #[serial] 123 | fn test_init_kanji_statistic() { 124 | drop_table("statistic").unwrap(); 125 | 126 | const KANJI: &str = "kanji"; 127 | init_kanji_statistic(KANJI).unwrap(); 128 | 129 | let statistic = load_kanji_statistic(KANJI).unwrap(); 130 | assert_eq!(statistic.count, 0); 131 | 132 | let statistic = load_kanji_statistic(KANJI).unwrap(); 133 | assert_eq!(statistic.count, 0); 134 | } 135 | 136 | #[test] 137 | #[serial] 138 | fn test_increment_kanji_statistic() { 139 | drop_table("statistic").unwrap(); 140 | 141 | const KANJI: &str = "kanji"; 142 | 143 | let statistic = increment_kanji_statistic(KANJI).unwrap(); 144 | assert_eq!(statistic.count, 1); 145 | 146 | let statistic = increment_kanji_statistic(KANJI).unwrap(); 147 | assert_eq!(statistic.count, 2); 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /src/database/mod.rs: -------------------------------------------------------------------------------- 1 | mod history_data; 2 | mod kanji_statistic; 3 | mod table; 4 | 5 | pub use history_data::load_full_history; 6 | pub use history_data::load_history_data; 7 | pub use history_data::store_ocr; 8 | pub use history_data::store_ocr_translation; 9 | pub use history_data::HistoryData; 10 | 11 | pub use kanji_statistic::increment_kanji_statistic; 12 | pub use kanji_statistic::init_kanji_statistic; 13 | pub use kanji_statistic::load_statistic; 14 | pub use kanji_statistic::KanjiStatistic; 15 | -------------------------------------------------------------------------------- /src/database/table.rs: -------------------------------------------------------------------------------- 1 | use std::{fs, path::PathBuf}; 2 | 3 | use anyhow::{Context, Ok, Result}; 4 | use rusqlite::Connection; 5 | 6 | const DATABASE_FILENAME: &str = if cfg!(test) { 7 | "manga_overlay_test.db" 8 | } else { 9 | "manga_overlay.db" 10 | }; 11 | 12 | pub fn create_database() -> Result { 13 | let output: PathBuf = get_output_path(DATABASE_FILENAME); 14 | 15 | Connection::open(&output).context("Could not create database") 16 | } 17 | 18 | pub fn create_table(create_table_query: &str) -> Result { 19 | let conn = create_database()?; 20 | 21 | conn.execute(create_table_query, []) 22 | .context("could not create table")?; 23 | 24 | Ok(conn) 25 | } 26 | 27 | #[cfg(test)] 28 | pub fn drop_table(table_name: &str) -> Result<()> { 29 | let conn = create_database()?; 30 | 31 | conn.execute(&format!("DROP TABLE IF EXISTS {table_name}"), [])?; 32 | 33 | Ok(()) 34 | } 35 | 36 | fn get_output_path(filename: &str) -> PathBuf { 37 | let path_buf = std::env::current_dir() 38 | .expect("unable to get current_dir") 39 | .join("output"); 40 | 41 | fs::create_dir_all(&path_buf).unwrap_or_else(|_| panic!("Unable to create output directory: {:?}", 42 | &path_buf)); 43 | 44 | path_buf.join(filename) 45 | } 46 | -------------------------------------------------------------------------------- /src/detect/comictextdetector.rs: -------------------------------------------------------------------------------- 1 | use std::sync::{Arc, LazyLock, Mutex}; 2 | 3 | use crate::detect::session_builder::create_session_builder; 4 | use anyhow::Result; 5 | use image::imageops::FilterType; 6 | use image::{DynamicImage, GenericImageView, Rgba}; 7 | use imageproc::drawing::draw_hollow_rect_mut; 8 | use imageproc::rect::Rect; 9 | use log::{debug, error}; 10 | use ndarray::Array4; 11 | use ort::session::Session; 12 | 13 | const INPUT_WIDTH: f32 = 1024.0; 14 | const INPUT_HEIGHT: f32 = 1024.0; 15 | 16 | pub static DETECT_STATE: LazyLock = LazyLock::new(DetectState::init); 17 | 18 | #[derive(Clone)] 19 | pub struct DetectState { 20 | pub session: Arc>>, 21 | } 22 | 23 | impl DetectState { 24 | pub fn init() -> Self { 25 | let data = load_model().ok(); 26 | let data = Mutex::new(data); 27 | let session = Arc::new(data); 28 | 29 | Self { session } 30 | } 31 | 32 | pub fn run_model(&self, threshold: f32, img: &mut DynamicImage) -> Vec { 33 | let model = self.session.lock().unwrap(); 34 | if let Some(model) = model.as_ref() { 35 | run_model(model, threshold, img).unwrap_or_else(|e| { 36 | error!("run_model error: {}", e); 37 | vec![] 38 | }) 39 | } else { 40 | vec![] 41 | } 42 | } 43 | } 44 | 45 | pub fn load_model() -> Result { 46 | let builder = create_session_builder()?; 47 | 48 | let detector_model = include_bytes!("../../resources/comictextdetector_blk.pt.onnx"); 49 | 50 | let session = builder.commit_from_memory(detector_model)?; 51 | Ok(session) 52 | } 53 | 54 | pub fn detect_boxes(model: &Session, original_img: &DynamicImage) -> Result> { 55 | let mut input = Array4::::zeros((1, 3, INPUT_WIDTH as usize, INPUT_HEIGHT as usize)); 56 | 57 | let img = original_img.resize_exact( 58 | INPUT_WIDTH as u32, 59 | INPUT_HEIGHT as u32, 60 | FilterType::CatmullRom, 61 | ); 62 | 63 | for pixel in img.pixels() { 64 | let x = pixel.0 as _; 65 | let y = pixel.1 as _; 66 | let [r, g, b, _] = pixel.2 .0; 67 | input[[0, 0, y, x]] = f32::from(r) / 255.; 68 | input[[0, 1, y, x]] = f32::from(g) / 255.; 69 | input[[0, 2, y, x]] = f32::from(b) / 255.; 70 | } 71 | 72 | // let outputs: SessionOutputs = model.run(ort::inputs!["images" => input.view()]?)?; 73 | let outputs = model.run(ort::inputs![input]?)?; 74 | 75 | let output_blk = outputs.get("blk").unwrap().try_extract_tensor::()?; 76 | 77 | let rows = output_blk 78 | .view() 79 | .axis_iter(ndarray::Axis(1)) 80 | .map(|row| Boxes::new(row.iter().copied().collect())) 81 | .collect(); 82 | 83 | Ok(rows) 84 | } 85 | 86 | #[derive(Clone, Debug)] 87 | pub struct Boxes { 88 | confidence: f32, 89 | x: f32, 90 | y: f32, 91 | w: f32, 92 | h: f32, 93 | } 94 | 95 | impl Boxes { 96 | fn new(row: Vec) -> Self { 97 | let x = (row[0] / INPUT_WIDTH).max(0.0); 98 | let y = (row[1] / INPUT_HEIGHT).max(0.0); 99 | let w = (row[2] / INPUT_WIDTH).max(0.0); 100 | let h = (row[3] / INPUT_HEIGHT).max(0.0); 101 | 102 | let confidence = row[4]; 103 | 104 | Self { 105 | confidence, 106 | x, 107 | y, 108 | w, 109 | h, 110 | } 111 | } 112 | 113 | fn get_top(&self) -> f32 { 114 | (self.y - self.h / 2.0).max(0.0) 115 | } 116 | 117 | fn get_bottom(&self) -> f32 { 118 | self.y + self.h / 2.0 119 | } 120 | 121 | fn get_left(&self) -> f32 { 122 | (self.x - self.w / 2.0).max(0.0) 123 | } 124 | 125 | fn get_right(&self) -> f32 { 126 | self.x + self.w / 2.0 127 | } 128 | 129 | fn overlaps(&self, other: &Boxes) -> bool { 130 | // if rectangle has area 0, no overlap 131 | if self.get_left() == self.get_right() 132 | || self.get_top() == self.get_bottom() 133 | || other.get_left() == other.get_right() 134 | || other.get_top() == other.get_bottom() 135 | { 136 | return false; 137 | } 138 | 139 | // If one rectangle is on left side of other 140 | if self.get_left() >= other.get_right() || other.get_left() >= self.get_right() { 141 | return false; 142 | } 143 | 144 | // If one rectangle is above other 145 | if self.get_top() >= other.get_bottom() || other.get_top() >= self.get_bottom() { 146 | return false; 147 | } 148 | 149 | true 150 | 151 | // Implement the logic to check if two boxes overlap 152 | } 153 | 154 | fn merge(&self, other: &Boxes) -> Boxes { 155 | // Implement the logic to merge two overlapping boxes into a combined box 156 | let min_left = self.get_left().min(other.get_left()); 157 | let min_top = self.get_top().min(other.get_top()); 158 | let max_right = self.get_right().max(other.get_right()); 159 | let max_bottom = self.get_bottom().max(other.get_bottom()); 160 | 161 | Boxes { 162 | confidence: (self.confidence + other.confidence) / 2.0, 163 | x: min_left + (max_right - min_left) / 2.0, 164 | y: min_top + (max_bottom - min_top) / 2.0, 165 | w: max_right - min_left, 166 | h: max_bottom - min_top, 167 | } 168 | } 169 | 170 | pub fn get_rect(&self, img: &DynamicImage) -> Rect { 171 | let img_width = img.width() as f32; 172 | let img_height = img.height() as f32; 173 | 174 | let x = self.get_left() * img_width; 175 | let y = self.get_top() * img_height; 176 | let width = self.w * img_width; 177 | let height = self.h * img_height; 178 | Rect::at(x as i32, y as i32).of_size(width as u32, height as u32) 179 | } 180 | } 181 | 182 | pub fn combine_overlapping_rects(boxes: Vec) -> Vec { 183 | let mut combined_boxes: Vec = vec![]; 184 | 185 | for next_box in boxes { 186 | let mut overlapped = false; 187 | for aggregate_box in &mut combined_boxes { 188 | if next_box.overlaps(aggregate_box) { 189 | *aggregate_box = aggregate_box.merge(&next_box); 190 | overlapped = true; 191 | } 192 | } 193 | if !overlapped { 194 | combined_boxes.push(next_box); 195 | } 196 | } 197 | 198 | combined_boxes 199 | } 200 | 201 | pub fn run_model(model: &Session, threshold: f32, img: &mut DynamicImage) -> Result> { 202 | debug!("detect_boxes..."); 203 | let mut boxes = detect_boxes(model, img)?; 204 | 205 | boxes.retain(|x| x.confidence > threshold); 206 | debug!("detect_boxes done with {}", boxes.len()); 207 | Ok(boxes) 208 | } 209 | 210 | pub fn draw_rects(img: &mut DynamicImage, boxes: &[Boxes]) { 211 | let red = Rgba([255, 0, 0, 255]); 212 | 213 | for row in boxes { 214 | let rect = row.get_rect(img); 215 | draw_hollow_rect_mut(img, rect, red); 216 | } 217 | } 218 | 219 | #[cfg(test)] 220 | mod tests { 221 | use super::*; 222 | use log::info; 223 | use std::path::Path; 224 | 225 | #[test] 226 | fn test_load() { 227 | let model = load_model().unwrap(); 228 | info!("Model loaded"); 229 | 230 | vec![0.0, 0.01, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] 231 | .into_iter() 232 | .enumerate() 233 | .for_each(|(i, conf)| { 234 | info!("Run {}", i); 235 | let res_dir = Path::new(env!("CARGO_MANIFEST_DIR")); 236 | let output = res_dir 237 | .join("output") 238 | .join(format!("output_{conf:.2}.jpg")); 239 | let input_path = res_dir.join("input").join("input.jpg"); 240 | let mut original_img = image::open(input_path.as_path()).unwrap(); 241 | 242 | let boxes = run_model(&model, conf, &mut original_img).unwrap(); 243 | 244 | draw_rects(&mut original_img, &boxes); 245 | 246 | let _ = original_img.save(&output); 247 | }); 248 | } 249 | } 250 | -------------------------------------------------------------------------------- /src/detect/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod comictextdetector; 2 | pub mod session_builder; -------------------------------------------------------------------------------- /src/detect/session_builder.rs: -------------------------------------------------------------------------------- 1 | use log::{info, warn}; 2 | use ort::execution_providers::{CUDAExecutionProvider, ExecutionProvider}; 3 | use ort::session::builder::SessionBuilder; 4 | use ort::session::Session; 5 | 6 | pub fn create_session_builder() -> anyhow::Result { 7 | let mut builder = Session::builder()? 8 | .with_optimization_level(ort::session::builder::GraphOptimizationLevel::Level3)? 9 | .with_intra_threads(4)?; 10 | 11 | let cuda = CUDAExecutionProvider::default(); 12 | if cuda.is_available()? { 13 | info!("CUDA is available"); 14 | } else { 15 | warn!("CUDA is not available"); 16 | } 17 | 18 | let result = cuda.register(&mut builder); 19 | if result.is_err() { 20 | warn!("Failed to register CUDA! {}", result.unwrap_err()); 21 | } else { 22 | info!("Registered CUDA"); 23 | } 24 | 25 | Ok(builder) 26 | } 27 | 28 | #[test] 29 | fn is_cuda_working() -> anyhow::Result<()> { 30 | let mut builder = Session::builder()? 31 | .with_optimization_level(ort::session::builder::GraphOptimizationLevel::Level3)? 32 | .with_intra_threads(4)?; 33 | 34 | let cuda = CUDAExecutionProvider::default(); 35 | assert!(cuda.is_available().is_ok()); 36 | 37 | let result = cuda.register(&mut builder); 38 | dbg!(&result); 39 | assert!(result.is_ok()); 40 | 41 | Ok(()) 42 | } -------------------------------------------------------------------------------- /src/jpn/dict.rs: -------------------------------------------------------------------------------- 1 | use crate::ui::shutdown::TASK_TRACKER; 2 | use jmdict::{Entry, KanjiElement, ReadingElement}; 3 | use multimap::MultiMap; 4 | use std::sync::LazyLock; 5 | 6 | const WINDOW_SIZE: usize = 50; 7 | const LARGEST_WORD_SIZE: usize = 15; 8 | const STEP_SIZE: usize = WINDOW_SIZE - LARGEST_WORD_SIZE; 9 | 10 | static JMDICT_MAP: LazyLock> = LazyLock::new(create_jmdict_map); 11 | 12 | fn create_jmdict_map() -> MultiMap { 13 | let x: Vec<(&'static str, Entry)> = jmdict::entries() 14 | .flat_map(|x| x.kanji_elements().map(move |e| (e.text, x))) 15 | .collect(); 16 | let y: Vec<(&'static str, Entry)> = jmdict::entries() 17 | .flat_map(|x| x.reading_elements().map(move |e| (e.text, x))) 18 | .collect(); 19 | 20 | let mut map: MultiMap = MultiMap::new(); 21 | for i in x { 22 | map.insert(i.0.chars().next().unwrap(), i.1); 23 | } 24 | 25 | for i in y { 26 | map.insert(i.0.chars().next().unwrap(), i.1); 27 | } 28 | 29 | map 30 | } 31 | 32 | pub async fn async_extract_words(input: &str) -> Vec<(String, Vec)> { 33 | let inter = input.chars().collect::>(); 34 | 35 | if inter.len() <= WINDOW_SIZE { 36 | return extract_words(input); 37 | } 38 | 39 | let mut windows: Vec = inter 40 | .windows(WINDOW_SIZE) 41 | .step_by(STEP_SIZE) 42 | .map(|x| x.iter().collect::()) 43 | .collect(); 44 | let window_char_count = windows.len() * STEP_SIZE; 45 | let remainder: String = input 46 | .chars() 47 | .skip(window_char_count.saturating_sub(LARGEST_WORD_SIZE)) 48 | .collect(); 49 | windows.push(remainder); 50 | 51 | let window_input: Vec<_> = windows 52 | .into_iter() 53 | .map(|x| TASK_TRACKER.spawn(async move { extract_words(&x) })) 54 | .collect(); 55 | 56 | let results: Vec)>> = futures::future::try_join_all(window_input) 57 | .await 58 | .unwrap_or_else(|e| { 59 | println!("async_extract_words: {e}"); 60 | vec![] 61 | }); 62 | 63 | combine_overlapping_vecs_with_entries(results) 64 | } 65 | 66 | fn combine_overlapping_vecs_with_entries( 67 | result_vecs: Vec)>>, 68 | ) -> Vec<(String, Vec)> { 69 | let mut buffer: Vec<(String, Vec)> = vec![]; 70 | let mut valid_until_index: usize = 0; 71 | 72 | let last_result_index = result_vecs.len() - 1; 73 | 74 | for (i, results) in result_vecs.into_iter().enumerate() { 75 | let mut offset = STEP_SIZE * i; 76 | 77 | let mut skip = 0; 78 | for (j, word) in results.iter().enumerate() { 79 | if offset == valid_until_index { 80 | skip = j; 81 | break; 82 | } 83 | let char_count = word.0.chars().count(); 84 | 85 | offset += char_count; 86 | } 87 | 88 | let results_length = if i >= last_result_index { 89 | results.len() 90 | } else { 91 | results.len() - 1 92 | }; 93 | let mut take: Vec<(String, Vec)> = results 94 | .into_iter() 95 | .take(results_length) 96 | .skip(skip) 97 | .collect(); 98 | valid_until_index += take.iter().map(|e| e.0.chars().count()).sum::(); 99 | 100 | buffer.append(&mut take); 101 | } 102 | buffer 103 | } 104 | 105 | pub fn remove_whitespace(s: &str) -> String { 106 | s.split_whitespace().collect() 107 | } 108 | 109 | fn extract_words(input: &str) -> Vec<(String, Vec)> { 110 | let mut output: Vec<(String, Vec)> = Vec::new(); 111 | let mut rest: Option<&str> = Some(input); 112 | while let Some(x) = rest { 113 | if x.is_empty() { 114 | return output; 115 | } 116 | 117 | let (prefix, matches) = extract_dict_entries(x); 118 | rest = x.strip_prefix(&prefix); 119 | 120 | output.push((prefix, matches)); 121 | } 122 | 123 | output 124 | } 125 | 126 | fn extract_dict_entries(input: &str) -> (String, Vec) { 127 | assert!(!input.is_empty(), "input '{input}'"); 128 | 129 | let mut current_prefix: String = input.chars().take(1).collect(); 130 | let initial_entries = JMDICT_MAP.get_vec(¤t_prefix.chars().next().unwrap()); 131 | if initial_entries.is_none() { 132 | return (current_prefix, vec![]); 133 | } 134 | 135 | let mut possible_matches: Vec = initial_entries.unwrap().clone(); 136 | if possible_matches.is_empty() { 137 | return (current_prefix, vec![]); 138 | } 139 | 140 | for i in 2..input.len() { 141 | let sub: String = input.chars().take(i).collect(); 142 | let new_matches = get_starting_matches(&sub, possible_matches.clone().into_iter()); 143 | 144 | if new_matches.is_empty() { 145 | return get_full_matches(current_prefix, possible_matches); 146 | } 147 | current_prefix = sub; 148 | possible_matches = new_matches; 149 | } 150 | 151 | get_full_matches(current_prefix, possible_matches) 152 | } 153 | 154 | fn get_full_matches(prefix: String, possible_matches: Vec) -> (String, Vec) { 155 | let full_matches: Vec = possible_matches 156 | .into_iter() 157 | .filter(|e| e.is_full_match(&prefix)) 158 | .collect(); 159 | 160 | (prefix, full_matches) 161 | } 162 | 163 | fn get_starting_matches(prefix: &str, entries: impl Iterator) -> Vec { 164 | entries.filter(|e| e.has_prefix(prefix)).collect() 165 | } 166 | 167 | trait HasText { 168 | fn get_text(&self) -> &'static str; 169 | } 170 | 171 | trait MatchesText { 172 | fn has_prefix(&self, prefix: &str) -> bool; 173 | fn is_full_match(&self, prefix: &str) -> bool; 174 | } 175 | 176 | impl MatchesText for T { 177 | fn has_prefix(&self, prefix: &str) -> bool { 178 | self.get_text().starts_with(prefix) 179 | } 180 | 181 | fn is_full_match(&self, prefix: &str) -> bool { 182 | self.get_text() == prefix 183 | } 184 | } 185 | 186 | impl HasText for ReadingElement { 187 | fn get_text(&self) -> &'static str { 188 | self.text 189 | } 190 | } 191 | 192 | impl HasText for KanjiElement { 193 | fn get_text(&self) -> &'static str { 194 | self.text 195 | } 196 | } 197 | 198 | impl MatchesText for Entry { 199 | fn has_prefix(&self, prefix: &str) -> bool { 200 | self.kanji_elements().any(|k| k.has_prefix(prefix)) 201 | } 202 | 203 | fn is_full_match(&self, prefix: &str) -> bool { 204 | self.kanji_elements().any(|k| k.is_full_match(prefix)) 205 | } 206 | } 207 | 208 | #[cfg(test)] 209 | mod tests { 210 | use std::time::SystemTime; 211 | 212 | use super::*; 213 | 214 | const LOREM : &str = "規ょフ記刊ねトゃ懸文朽っ面務75載ユ対芸フルラ寄63分ちょと対本1張スヘワツ大覧げんち語世び職学ヒヨフ報断ネケホ盟工フトミ開査亜才きほあ。例キネヒユ旅揮あれ況柱ッしわひ剤平さ注分投天タウヤ支警うイほさ考広もび施甲マニテタ告容イじ版提聞チ幅行ミニヒル属内て任喜らラよ着集輝れ冷済成索のでつ。 215 | 216 | 督だょ職真ばを確辺ぐ碁近ネ然有タラリオ未3備月ラノムテ員職トね録記ご選図コフイ史経82置リフ湯震ムシリタ展査テ清面をト格9検め。1同勢ト形界めり禁私メヒア航移だとせ昇分革会上ミイ感築わっば事購おリフ生人シヌタ残革書ゅリ委何ヱマ従写ヲノヤネ都地みろ意携をん月男妊ね。 217 | 218 | 大エヲモ別意ユタセテ指車載城さ影真ラ界年じフうめ一子葉けラえだ者質ょずせ研言アロスリ迎村ゃ決欺茶針促さよば。果ハ週7効ご読失転探とめみリ婚71常ねあべ文式セ京討そばス育望ツエ訴5村びン医僕滞硬イルッた。89情モハエ顔書素ミ求動ぱ供先ざをトル宣択ぼ館聞ごへな扶観ほもぞト今合ヘモコ見費ナミ理発ぐふ州7過掛海ま頭型ルサフメ投要サリメル持務れほ威悲カ判覇しすは。 219 | 220 | 後ぼ旅他がつル人宮めはに研最ドやじ小情新むぱにっ元亡ネケ論都磨ア屋永覧橋びいあ術21編クトキ庁体みるを作71惑はスづ始一ノフヲ無運ラリこふ。理ろわ真広以クヒ思撮1化4著ホムヘ京芸るだ応氷ンルふ刑勝スみフめ私作ユウコ出更び伝露キシ月断メマシ応根企かねす朝慶レコセ今価ル山子ねみべそ。 221 | 222 | 載えすめ太軒つでゅン読方ヤウ関消ずスば優載ど成日目リ広各さ伊選メアタウ直7水ゃ古検スヒ育読イセヒソ聞63報るゃつ覧裁つちゅぜ記馬の。終撃トぐほ世凍ホチ前内マハ寄敵コ信2違ヱヤヘロ恋第ソテ見中車せスえ始音細へ経警べぎ選卸す。高57作才ニソノ除家ずク鮮不のけえス欺別出湘ほび理軍ごラぜ朗皇がこへ総幕ヒ不本オクイ改地トノ何能3般セサラ図都ムヤハテ捕仙沢温ひぐえ。 223 | 224 | 体子っをさ変質ツチヒロ新害トなあ倍上サ駒誰ふ込験ルソハ下堀なじよゆ資之ユ月9問ミメケ止苗きフぼ者載ど長真嚇クぞ生書マヲ使幅採べめぐじ。療をとば森省ぽく竹月物せいほぶ速属切っ更94告算京20聞ヌ値読然ヲネ紀未アヱ荒読転スイ告与ほっッ委天条ヤキヘ軍機健了つ。絶3北ナ量43説れつ器教つン常牲むあス利経ロエユ断過リ国彫記ゆひあ支光へがれぴ子気じ伊化ヱスラ備偉塔にて。 225 | 226 | 書ぱふうつ和部だ愛根ろ位館定レ増気ーぽ止8読ヱスリオ号社ヨケミノ験盛るルほ日記べま官横ゅがゃげ黒協外せ勝浦ヨス申真ねゅ朝入殺ぜかさ載康メ視周おっが。転ー一菊セノロ年川ツウナフ天京メヱ施96連東ふ責平能そでほ覧公ヲルソナ事機ゃ特74高無旗昼栗びぜ。察んそ供遺ッわにみ医夢ユ願親ラセヘキ少識ナ韓疑時シコネテ強男ワネリ研効とょ球9加ッし給覚格8隊セ集乳クあリづ。 227 | 228 | 4耕クコ町74選タ崎浦権長そっざ厳左ルメ問42台会ナトア策軽だつ生佐ヱカ多千政伎券ぜ。医レ聞止的ろづ供明ケ提明ノネイセ推整オケア会禁ホユ藤覧フイ資谷さ川5初コエノ社96知辞たしくぶ済遺拡よじお。攻チヲユ小国イ材関理け父化画ヨナミ語笑ざこ神之えるはう終垂んスせな要遺も届志ヌタ初日ドをろた歌応ざ変一え要天ロフ刊変税はあぞ回界つ実円紹へたれ可伝拓泰至こおに。 229 | 230 | 石オヱツ指1車ゆラ軽明ぶめた喰周おて起研禁際ゆちーだ刊政ミヱネヌ知服今ろひ稿応のあふ今内選イぽッ写就覧喜ふろみ。教キノ年13革まど全記じリさ講中アネ書2全テラヘ青近崎てすゃ出71引ウレフユ首代そす禁自書まーぽ雪保ヤテヒ防景ヒリ長韓ノクフヲ利止叟噂愉びどむ。人ぶげなつ愛重ドろ催五キ詳短移アヒ折泰ケル開塊ぎぼゅ企8意囲まゅめみ産選あてリ障男長ラヲ北瀬セ入成販ょすは。 231 | 232 | 三業オネ各政タホ技九づッン題任ノリ載75左ゅとのあ豆条必野きりゅ一際最ナアカロ高8著ンごイな区港まさ日天よびド収金ょぽ。睦べむクふ実93家福ウツヘ競満万キハモソ長投せ強巨そ観条マセ速能続ぶづの使保ゆ試町ラア江雑コナ福富開王乏えか。悪どぜとせ遺意志ムヒ事経からス真取ぴぐっ芸験ざ闘調たざへ広上ぶ聞題メワテヘ阜13家ネサ家秋ラ経都チメヨ職左削幸績よし。"; 233 | 234 | #[tokio::test(flavor = "multi_thread")] 235 | async fn benchmark() { 236 | let input = LOREM.repeat(2); 237 | 238 | for _ in 0..10 { 239 | let label = "async_extract_words"; 240 | let start = SystemTime::now(); 241 | let _ = async_extract_words(&input).await; 242 | let end = SystemTime::now(); 243 | let duration = end.duration_since(start).unwrap(); 244 | println!("function took {label} {duration:?}"); 245 | 246 | println!("----"); 247 | } 248 | } 249 | } 250 | -------------------------------------------------------------------------------- /src/jpn/kanji.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use serde_with::{serde_as, DefaultOnNull}; 3 | use std::collections::HashMap; 4 | use std::sync::LazyLock; 5 | 6 | static KANJI_MAP: LazyLock> = 7 | LazyLock::new(|| get_map_from_json(include_str!("kanji.json"))); 8 | 9 | #[serde_as] 10 | #[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Debug, Default)] 11 | #[serde(default)] 12 | pub struct KanjiData { 13 | pub strokes: u8, 14 | pub grade: Option, 15 | pub freq: Option, 16 | pub jlpt_old: Option, 17 | pub jlpt_new: Option, 18 | pub meanings: Vec, 19 | pub readings_on: Vec, 20 | pub readings_kun: Vec, 21 | pub wk_level: Option, 22 | #[serde_as(deserialize_as = "DefaultOnNull")] 23 | pub wk_meanings: Vec, 24 | #[serde_as(deserialize_as = "DefaultOnNull")] 25 | pub wk_readings_on: Vec, 26 | #[serde_as(deserialize_as = "DefaultOnNull")] 27 | pub wk_readings_kun: Vec, 28 | #[serde_as(deserialize_as = "DefaultOnNull")] 29 | pub wk_radicals: Vec, 30 | } 31 | 32 | #[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Debug)] 33 | pub struct KanjiMap { 34 | map: HashMap, 35 | } 36 | 37 | pub fn get_kanji_data(word: char) -> Option { 38 | KANJI_MAP.get(&word).cloned() 39 | } 40 | 41 | fn get_map_from_json(json: &str) -> HashMap { 42 | serde_json::from_str(json).unwrap() 43 | } 44 | 45 | #[cfg(test)] 46 | mod tests { 47 | use super::*; 48 | use crate::jpn::JpnWordInfo; 49 | use log::info; 50 | 51 | #[test] 52 | fn typed_example() -> Result<(), ()> { 53 | // Some JSON input data as a &str. Maybe this comes from the user. 54 | // let data = std::include_str!("kanji.json"); 55 | 56 | // Parse the string of data into a Person object. This is exactly the 57 | // same function as the one that produced serde_json::Value above, but 58 | // now we are asking it for a Person as output. 59 | // let string = fs::read_to_string("/kanji.json").unwrap(); 60 | // get_map_from_json(&string); 61 | 62 | let word = "唖".chars().next().unwrap(); 63 | let option = KANJI_MAP.get(&word).unwrap(); 64 | info!("{:#?}", option); 65 | 66 | let kanji = serde_json::to_string(option).unwrap(); 67 | 68 | info!("{}", kanji); 69 | 70 | let info = JpnWordInfo::new(word); 71 | 72 | let kanji = serde_json::to_string(&info).unwrap(); 73 | 74 | info!("{}", kanji); 75 | 76 | Ok(()) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/jpn/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::jpn::kanji::{get_kanji_data, KanjiData}; 2 | use crate::ui::shutdown::TASK_TRACKER; 3 | use jmdict::{Entry, GlossLanguage}; 4 | 5 | pub mod dict; 6 | pub mod kanji; 7 | 8 | #[derive(Debug, serde::Serialize, serde::Deserialize, Default, PartialEq, Clone)] 9 | #[serde(default)] 10 | pub struct JpnData { 11 | pub words: Vec, 12 | pub jm_dict: Vec, 13 | } 14 | 15 | impl JpnData { 16 | fn new(word: &str, entries: &[Entry]) -> Self { 17 | let words = word.chars().map(JpnWordInfo::new).collect(); 18 | 19 | let jm_dict = entries.iter().map(JmDictInfo::new).collect(); 20 | 21 | Self { words, jm_dict } 22 | } 23 | 24 | pub fn has_kanji_data(&self) -> bool { 25 | self.words.iter().any(|w| w.kanji_data.is_some()) 26 | || self.jm_dict.iter().any(|w| !w.info.is_empty()) 27 | } 28 | 29 | pub fn get_kanji(&self) -> String { 30 | self.words.iter().map(|x| x.word).collect() 31 | } 32 | 33 | pub fn get_info_rows(&self) -> Vec { 34 | if self.words.is_empty() { 35 | return vec![]; 36 | } 37 | 38 | let mut info = vec![]; 39 | 40 | self.jm_dict 41 | .iter() 42 | .for_each(|x| info.extend(x.info.iter().cloned())); 43 | 44 | self.words 45 | .iter() 46 | .filter(|x| { 47 | x.kanji_data 48 | .as_ref() 49 | .is_some_and(|x| !x.meanings.is_empty()) 50 | }) 51 | .map(|x| { 52 | [ 53 | format!( 54 | "{}: {}", 55 | x.word, 56 | x.kanji_data.as_ref().unwrap().meanings.join(", ") 57 | ), 58 | format!( 59 | "on Reading: {}", 60 | x.kanji_data.as_ref().unwrap().readings_on.join(", ") 61 | ), 62 | format!( 63 | "kun Reading: {}", 64 | x.kanji_data.as_ref().unwrap().readings_kun.join(", ") 65 | ), 66 | ] 67 | }) 68 | .for_each(|x| info.extend(x.iter().cloned())); 69 | 70 | info.retain(|x| !x.is_empty()); 71 | 72 | info 73 | } 74 | } 75 | 76 | #[derive(Debug, serde::Serialize, serde::Deserialize, Default, PartialEq, Clone)] 77 | #[serde(default)] 78 | pub struct JpnWordInfo { 79 | pub word: char, 80 | #[serde(skip)] 81 | pub kanji_data: Option, 82 | } 83 | 84 | impl JpnWordInfo { 85 | fn new(word: char) -> Self { 86 | let kanji_data = get_kanji_data(word); 87 | 88 | Self { word, kanji_data } 89 | } 90 | } 91 | 92 | #[derive(Debug, serde::Serialize, serde::Deserialize, Default, PartialEq, Clone)] 93 | #[serde(default)] 94 | pub struct JmDictInfo { 95 | pub info: Vec, 96 | } 97 | 98 | impl JmDictInfo { 99 | fn new(entry: &Entry) -> Self { 100 | let info: Vec = get_info_from_entry(entry).into_iter().collect(); 101 | Self { info } 102 | } 103 | } 104 | 105 | pub async fn get_jpn_data(input: &str) -> Vec> { 106 | let lines: Vec = input.lines().map(dict::remove_whitespace).collect(); 107 | 108 | let window_input: Vec<_> = lines 109 | .into_iter() 110 | .map(|x| { 111 | TASK_TRACKER.spawn(async move { 112 | dict::async_extract_words(&x) 113 | .await 114 | .iter() 115 | .map(|(txt, entries)| JpnData::new(txt, entries)) 116 | .collect() 117 | }) 118 | }) 119 | .collect(); 120 | 121 | let results: Vec> = futures::future::try_join_all(window_input) 122 | .await 123 | .unwrap_or_default(); 124 | 125 | results 126 | } 127 | 128 | pub fn get_info_from_entry(e: &Entry) -> Vec { 129 | let mut output: Vec = Vec::new(); 130 | for kanji in e.kanji_elements() { 131 | output.push(format!("Kanji: {:?}, ", kanji.text.to_string())); 132 | } 133 | 134 | for reading in e.reading_elements() { 135 | output.push(format!("Reading: {:?}, ", reading.text.to_string())); 136 | for info in reading.infos() { 137 | output.push(format!("{info:?}, ")); 138 | } 139 | } 140 | output.push(String::new()); 141 | 142 | for (index, sense) in e.senses().enumerate() { 143 | let parts_of_speech = sense 144 | .parts_of_speech() 145 | .map(|part| format!("{part}")) 146 | .collect::>() 147 | .join(", "); 148 | let english_meaning = sense 149 | .glosses() 150 | .filter(|g| g.language == GlossLanguage::English) 151 | .map(|g| g.text) 152 | .collect::>() 153 | .join("; "); 154 | output.push(format!( 155 | "{}. {}: {}", 156 | index + 1, 157 | parts_of_speech, 158 | english_meaning 159 | )); 160 | 161 | for info in sense.topics() { 162 | output.push(format!("{info:?}, ")); 163 | } 164 | } 165 | 166 | output 167 | } 168 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![warn(clippy::all, rust_2018_idioms)] 2 | #![allow( 3 | clippy::must_use_candidate, 4 | clippy::module_name_repetitions, 5 | clippy::cast_possible_truncation, 6 | clippy::cast_sign_loss, 7 | clippy::cast_precision_loss, 8 | clippy::float_cmp 9 | )] 10 | mod ui; 11 | 12 | use action::ScreenshotParameter; 13 | pub use ui::app::OcrApp; 14 | 15 | use anyhow::{Context, Ok, Result}; 16 | use image::{DynamicImage, RgbaImage}; 17 | use rusty_tesseract::Args; 18 | use screenshots::Screen; 19 | 20 | pub(crate) mod action; 21 | pub(crate) mod database; 22 | pub(crate) mod detect; 23 | pub(crate) mod jpn; 24 | pub(crate) mod ocr; 25 | pub(crate) mod translation; 26 | 27 | impl ScreenshotParameter { 28 | pub fn get_screenshot(&self) -> Result { 29 | let screen = Screen::from_point(self.x, self.y)?; 30 | let image = screen.capture_area( 31 | self.x - screen.display_info.x, 32 | self.y - screen.display_info.y, 33 | self.width, 34 | self.height, 35 | )?; 36 | 37 | let bytes = image.to_vec(); 38 | let image = RgbaImage::from_raw(image.width(), image.height(), bytes) 39 | .context("screenshot failed")?; 40 | 41 | Ok(DynamicImage::ImageRgba8(image)) 42 | } 43 | } 44 | 45 | pub struct OcrParameter { 46 | pub args: Args, 47 | } 48 | 49 | #[derive(Debug, Clone, PartialEq)] 50 | pub struct OcrResult { 51 | pub ocr: String, 52 | pub confidence: f32, 53 | pub rects: Vec, 54 | } 55 | 56 | #[derive(Debug, Clone, PartialEq)] 57 | pub struct OcrRect { 58 | symbol: String, 59 | top: i32, 60 | left: i32, 61 | width: i32, 62 | height: i32, 63 | } 64 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | #![warn(clippy::all, rust_2018_idioms)] 2 | #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] 3 | use log4rs::config::Deserializers; 4 | use manga_overlay::OcrApp; 5 | use std::{fs, path::Path}; 6 | 7 | #[tokio::main] 8 | async fn main() -> eframe::Result { 9 | init_logger(); 10 | 11 | let native_options = eframe::NativeOptions { 12 | viewport: egui::ViewportBuilder::default() 13 | .with_transparent(true) 14 | .with_always_on_top() 15 | .with_min_inner_size([300.0, 220.0]) 16 | .with_icon( 17 | eframe::icon_data::from_png_bytes(&include_bytes!("../resources/icon-256.png")[..]) 18 | .expect("Failed to load icon"), 19 | ), 20 | ..Default::default() 21 | }; 22 | eframe::run_native( 23 | "Manga Overlay", 24 | native_options, 25 | Box::new(|cc| Ok(Box::new(OcrApp::new(cc)))), 26 | ) 27 | } 28 | 29 | const LOG_CONFIG_DIR: &str = "config"; 30 | const LOG_CONFIG: &str = "config/log4rs.yaml"; 31 | 32 | fn init_logger() { 33 | fs::create_dir_all(LOG_CONFIG_DIR).expect("Config directory creation failed"); 34 | if !Path::new(&LOG_CONFIG).exists() { 35 | fs::write(LOG_CONFIG, include_str!("../config/log4rs.yaml")) 36 | .expect("Config file creation failed"); 37 | } 38 | 39 | log4rs::init_file("config/log4rs.yaml", Deserializers::default()).expect("Logger init failed"); 40 | } 41 | -------------------------------------------------------------------------------- /src/ocr/manga_ocr.rs: -------------------------------------------------------------------------------- 1 | use crate::detect::session_builder::create_session_builder; 2 | use hf_hub::api::sync::Api; 3 | use image::DynamicImage; 4 | use itertools::Itertools; 5 | use ndarray::{s, stack, Array3, Array4, ArrayBase, Axis, Dim, Ix, OwnedRepr}; 6 | use ort::{inputs, session::Session}; 7 | use std::ops::{Div, Sub}; 8 | use std::sync::{Arc, LazyLock, Mutex}; 9 | 10 | type MangaOcrState = Arc>>; 11 | 12 | pub static MANGA_OCR: LazyLock = 13 | LazyLock::new(|| Arc::new(Mutex::new(MangaOCR::new()))); 14 | 15 | #[derive(Debug)] 16 | pub struct MangaOCR { 17 | model: Session, 18 | vocab: Vec, 19 | } 20 | 21 | impl MangaOCR { 22 | pub fn new() -> anyhow::Result { 23 | let api = Api::new()?; 24 | let repo = api.model("mayocream/koharu".to_string()); 25 | let model_path = repo.get("manga-ocr.onnx")?; 26 | let vocab_path = repo.get("vocab.txt")?; 27 | 28 | let builder = create_session_builder()?; 29 | 30 | let model = builder.commit_from_file(model_path)?; 31 | 32 | let vocab = std::fs::read_to_string(vocab_path) 33 | .map_err(|e| anyhow::anyhow!("Failed to read vocab file: {e}"))? 34 | .lines() 35 | .map(|s| s.to_string()) 36 | .collect::>(); 37 | 38 | Ok(Self { model, vocab }) 39 | } 40 | 41 | pub fn inference(&self, images: &[DynamicImage]) -> anyhow::Result> { 42 | if images.is_empty() { 43 | return Ok(vec![]); 44 | } 45 | 46 | let batch_size = images.len(); 47 | let tensor = Self::create_image_tensor(images); 48 | 49 | let token_ids = self.get_token_ids(batch_size, tensor)?; 50 | 51 | let texts = token_ids.iter().map(|x| self.decode_tokens(x)).collect(); 52 | Ok(texts) 53 | } 54 | 55 | 56 | fn decode_tokens(&self, token_ids: &Vec) -> String { 57 | let text = token_ids 58 | .iter() 59 | .filter(|&&id| id >= 5) 60 | .filter_map(|&id| self.vocab.get(id as usize).cloned()) 61 | .collect::>(); 62 | 63 | text.join("") 64 | } 65 | 66 | fn get_token_ids( 67 | &self, 68 | batch_size: usize, 69 | tensor: ArrayBase, Dim<[Ix; 4]>>, 70 | ) -> anyhow::Result>> { 71 | let mut done_state: Vec = vec![false; batch_size]; 72 | let mut token_ids: Vec> = vec![vec![2i64]; batch_size]; // Start token 73 | 74 | 'outer: for _ in 0..300 { 75 | // Create input tensors 76 | let input = ndarray::Array::from_shape_vec( 77 | (batch_size, token_ids[0].len()), 78 | token_ids.iter().flatten().cloned().collect(), 79 | )?; 80 | let inputs = inputs! { 81 | "image" => tensor.view(), 82 | "token_ids" => input, 83 | }?; 84 | 85 | // Run inference 86 | let outputs = self.model.run(inputs)?; 87 | 88 | // Extract logits from output 89 | let logits = outputs["logits"].try_extract_tensor::()?; 90 | 91 | // Get last token logits and find argmax 92 | let logits_view = logits.view(); 93 | 94 | for i in 0..batch_size { 95 | if done_state[i] { 96 | token_ids[i].push(3); 97 | continue; 98 | } 99 | 100 | let last_token_logits = logits_view.slice(s![i, -1, ..]); 101 | 102 | let (token_id, _) = last_token_logits 103 | .iter() 104 | .enumerate() 105 | .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap()) 106 | .unwrap_or((0, &0.0)); 107 | 108 | token_ids[i].push(token_id as i64); 109 | 110 | // Break if end token 111 | if token_id as i64 == 3 { 112 | done_state[i] = true; 113 | 114 | if done_state.iter().all(|&x| x) { 115 | break 'outer; 116 | } 117 | } 118 | } 119 | } 120 | Ok(token_ids) 121 | } 122 | 123 | fn create_image_tensor(images: &[DynamicImage]) -> Array4 { 124 | let arrays = images.iter().map(|x| Self::fast_image_to_ndarray(x)).collect_vec(); 125 | let stack = Self::join_arrays_stack(&arrays); 126 | 127 | stack 128 | } 129 | 130 | fn fast_image_to_ndarray(img: &DynamicImage) -> Array3 { 131 | let img = img.grayscale().to_rgb8(); 132 | let img = image::imageops::resize(&img, 224, 224, image::imageops::FilterType::Lanczos3); 133 | 134 | let (width, height) = img.dimensions(); 135 | let raw_buf = img.as_raw(); 136 | 137 | let array = Array3::from_shape_vec((height as usize, width as usize, 3), 138 | raw_buf.iter().map(|&x| x as f32).collect()) 139 | .unwrap().div(255.0).sub(0.5).div(0.5); 140 | 141 | array 142 | } 143 | 144 | fn join_arrays_stack(arrays: &[Array3]) -> Array4 { 145 | let views: Vec<_> = arrays.iter().map(|a| a.view().permuted_axes([2, 0, 1])).collect(); 146 | stack(Axis(0), &views).unwrap() 147 | } 148 | } -------------------------------------------------------------------------------- /src/ocr/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::ocr::manga_ocr::MANGA_OCR; 2 | use anyhow::Result; 3 | use image::DynamicImage; 4 | use serde::{Deserialize, Serialize}; 5 | use strum::{EnumIter, EnumString}; 6 | 7 | pub mod manga_ocr; 8 | 9 | #[derive(Debug, Clone, PartialEq, strum::Display, EnumString, EnumIter, Serialize, Deserialize)] 10 | pub enum OcrBackend { 11 | #[strum(ascii_case_insensitive)] 12 | MangaOcr, 13 | } 14 | 15 | impl OcrBackend { 16 | pub fn run_backends(images: &[DynamicImage], backends: &[OcrBackend]) -> Vec { 17 | let backend_count: usize = backends.len(); 18 | 19 | let backend_outputs: Vec> = backends 20 | .iter() 21 | .map(|e| (e, e.run_ocr(&images))) 22 | .map(|e| concat_backend_output(e.0, e.1, backend_count)) 23 | .collect(); 24 | 25 | let mut output: Vec = vec![]; 26 | for (i, backend_output) in backend_outputs.iter().enumerate() { 27 | if i == 0 { 28 | output.clone_from(backend_output); 29 | } else { 30 | output = output 31 | .into_iter() 32 | .zip(backend_output.iter()) 33 | .map(|x| [x.0, x.1.to_string()].join("\n\n").trim().to_string()) 34 | .collect(); 35 | } 36 | } 37 | 38 | output 39 | } 40 | 41 | pub fn run_ocr(&self, images: &[DynamicImage]) -> Result> { 42 | if images.is_empty() { 43 | return Ok(vec![]); 44 | } 45 | 46 | match self { 47 | OcrBackend::MangaOcr => Ok(run_manga_ocr(images)), 48 | } 49 | } 50 | } 51 | 52 | fn run_manga_ocr(images: &[DynamicImage]) -> Vec { 53 | let model = MANGA_OCR.lock().unwrap(); 54 | if let Ok(model) = model.as_ref() { 55 | return model.inference(images).unwrap(); 56 | } 57 | vec![] 58 | } 59 | 60 | fn concat_backend_output( 61 | backend: &OcrBackend, 62 | output: Result>, 63 | backend_count: usize, 64 | ) -> Vec { 65 | let outputs = output.unwrap_or_else(|e| vec![e.to_string()]); 66 | outputs 67 | .into_iter() 68 | .map(|x| { 69 | if backend_count > 1 { 70 | [backend.to_string(), x].join("\n") 71 | } else { 72 | x 73 | } 74 | }) 75 | .collect() 76 | } 77 | 78 | #[cfg(test)] 79 | mod tests { 80 | use log::info; 81 | 82 | use crate::action::{run_ocr, ResultData, ScreenshotParameter, ScreenshotResult}; 83 | use crate::ocr::OcrBackend; 84 | use crate::ocr::OcrBackend::MangaOcr; 85 | 86 | #[test] 87 | fn ocr_backend_serialize() { 88 | let backends: Vec = vec![MangaOcr]; 89 | 90 | let json = serde_json::to_string(&backends).unwrap(); 91 | info!("json: {}", json); 92 | assert_eq!(json, r#"["MangaOcr"]"#); 93 | 94 | let result: Vec = serde_json::from_str(&json).unwrap(); 95 | info!("parsed: {:?}", result); 96 | assert_eq!(backends, result); 97 | } 98 | 99 | #[tokio::test] 100 | async fn test_detect_boxes_and_manga_ocr() { 101 | let expected = vec![ 102 | ResultData { 103 | x: 565, 104 | y: 159, 105 | w: 96, 106 | h: 131, 107 | ocr: "今年はいいことがありそうだ。".to_string(), 108 | ..Default::default() 109 | }, 110 | ResultData { 111 | x: 749, 112 | y: 205, 113 | w: 63, 114 | h: 155, 115 | ocr: "のどかなお正月だなあ。".to_string(), 116 | ..Default::default() 117 | }, 118 | ResultData { 119 | x: 758, 120 | y: 711, 121 | w: 94, 122 | h: 92, 123 | ocr: "四十分後火あぶりなる。".to_string(), 124 | ..Default::default() 125 | }, 126 | ResultData { 127 | x: 121, 128 | y: 717, 129 | w: 67, 130 | h: 84, 131 | ocr: "出てこいつ。".to_string(), 132 | ..Default::default() 133 | }, 134 | ResultData { 135 | x: 437, 136 | y: 727, 137 | w: 83, 138 | h: 75, 139 | ocr: "だれだへんないうや".to_string(), 140 | ..Default::default() 141 | }, 142 | ResultData { 143 | x: 100, 144 | y: 102, 145 | w: 111, 146 | h: 81, 147 | ocr: "いやあ、ろくなことがないね。".to_string(), 148 | ..Default::default() 149 | }, 150 | ResultData { 151 | x: 60, 152 | y: 403, 153 | w: 130, 154 | h: 124, 155 | ocr: "野比のび太は三十分後に道をつる。".to_string(), 156 | ..Default::default() 157 | }, 158 | ]; 159 | 160 | run_test(&expected).await; 161 | } 162 | 163 | async fn run_test(expected: &[ResultData]) { 164 | let image = image::open("input/input.jpg").expect("Failed to open image"); 165 | let run_ocr: ScreenshotResult = run_ocr( 166 | ScreenshotParameter { 167 | detect_boxes: true, 168 | backends: vec![OcrBackend::MangaOcr], 169 | ..ScreenshotParameter::default() 170 | }, 171 | image, 172 | ) 173 | .await 174 | .unwrap(); 175 | 176 | run_ocr 177 | .ocr_results 178 | .iter() 179 | .zip(expected.iter()) 180 | .for_each(|(a, b)| { 181 | test_result_data(a, b); 182 | }); 183 | } 184 | 185 | fn test_result_data(a: &ResultData, b: &ResultData) { 186 | assert_eq!(a.x, b.x); 187 | assert_eq!(a.y, b.y); 188 | assert_eq!(a.w, b.w); 189 | assert_eq!(a.h, b.h); 190 | assert_eq!(a.ocr, b.ocr); 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /src/translation/google.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{anyhow, Result}; 2 | use itertools::Itertools; 3 | 4 | pub async fn translate(jpn_text: &str) -> Result { 5 | let url = "https://translate.google.com/m?sl=ja&tl=en&hl=en"; 6 | let response = reqwest::get(format!("{url}&q={jpn_text}")).await?; 7 | let body = response.text().await?; 8 | 9 | let document = scraper::Html::parse_document(&body); 10 | 11 | let selector = scraper::Selector::parse("div.result-container") 12 | .map_err(|_| anyhow!("div.result-container selector not found"))?; 13 | let translation = document.select(&selector).map(|x| x.inner_html()).join(""); 14 | Ok(translation) 15 | } 16 | 17 | #[cfg(test)] 18 | mod tests { 19 | use super::*; 20 | use log::info; 21 | 22 | #[tokio::test] 23 | async fn test_request_google() { 24 | let body = translate("今 いま 私 わたし\n は 東京 とうきょう に 住 す んでいるので") 25 | .await 26 | .unwrap(); 27 | info!("{}", body); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/translation/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod google; 2 | -------------------------------------------------------------------------------- /src/ui/app.rs: -------------------------------------------------------------------------------- 1 | use super::background_rect::BackgroundRect; 2 | use super::kanji_history_ui::{init_history_updater, HistoryDataUi}; 3 | use super::kanji_statistic_ui::{init_kanji_statistic_updater, KanjiStatisticUi}; 4 | use super::settings::{AppSettings, Backend, BackendStatus}; 5 | use crate::detect::comictextdetector::DETECT_STATE; 6 | use crate::ocr::manga_ocr::MANGA_OCR; 7 | use crate::ui::event::Event::UpdateBackendStatus; 8 | use crate::ui::event::EventHandler; 9 | use crate::ui::shutdown::{shutdown_tasks, TASK_TRACKER}; 10 | use egui::Context; 11 | use futures::join; 12 | use std::sync::LazyLock; 13 | 14 | #[derive(serde::Deserialize, serde::Serialize, Default)] 15 | #[serde(default)] 16 | pub struct OcrApp { 17 | pub settings: AppSettings, 18 | pub background_rect: BackgroundRect, 19 | pub kanji_statistic: KanjiStatisticUi, 20 | pub history: HistoryDataUi, 21 | } 22 | 23 | impl OcrApp { 24 | pub fn new(cc: &eframe::CreationContext<'_>) -> Self { 25 | let ocr_app: Self = if let Some(storage) = cc.storage { 26 | let storage: Self = eframe::get_value(storage, eframe::APP_KEY).unwrap_or_default(); 27 | 28 | let ctx = &cc.egui_ctx; 29 | 30 | init_font(ctx); 31 | ctx.send_viewport_cmd(egui::ViewportCommand::Decorations( 32 | storage.settings.decorations, 33 | )); 34 | 35 | storage 36 | } else { 37 | Default::default() 38 | }; 39 | 40 | init_history_updater(cc.egui_ctx.clone()); 41 | init_kanji_statistic_updater(cc.egui_ctx.clone()); 42 | 43 | Self::init_backends(&cc.egui_ctx); 44 | 45 | ocr_app 46 | } 47 | 48 | pub fn init_backends(ctx: &Context) { 49 | let ctx1 = ctx.clone(); 50 | TASK_TRACKER.spawn(async move { 51 | let init1 = TASK_TRACKER.spawn(async { LazyLock::force(&MANGA_OCR) }); 52 | let init2 = TASK_TRACKER.spawn(async { LazyLock::force(&DETECT_STATE) }); 53 | let (result1, result2) = join!(init1, init2); 54 | 55 | ctx1.emit(UpdateBackendStatus( 56 | Backend::MangaOcr, 57 | if result1.is_ok() && result2.is_ok() { 58 | BackendStatus::Ready 59 | } else { 60 | BackendStatus::Error 61 | }, 62 | )); 63 | }); 64 | } 65 | 66 | fn show(&mut self, ctx: &Context) { 67 | if ctx.input(|i| i.viewport().close_requested()) { 68 | shutdown_tasks(); 69 | ctx.send_viewport_cmd(egui::ViewportCommand::Visible(false)); 70 | } 71 | 72 | self.background_rect.show(ctx, &self.settings); 73 | 74 | self.settings.show(ctx); 75 | 76 | if self.settings.show_statistics { 77 | self.kanji_statistic.show(ctx); 78 | } 79 | if self.settings.show_history { 80 | self.history.show(ctx); 81 | } 82 | 83 | self.update_mouse_passthrough(ctx); 84 | 85 | if self.settings.show_debug_cursor { 86 | self.draw_mouse_position(ctx); 87 | } 88 | } 89 | } 90 | 91 | impl eframe::App for OcrApp { 92 | /// Called each time the UI needs repainting, which may be many times per second. 93 | fn update(&mut self, ctx: &Context, _frame: &mut eframe::Frame) { 94 | ctx.update_state(self); 95 | 96 | ctx.set_zoom_factor(self.settings.zoom_factor); 97 | 98 | self.show(ctx); 99 | } 100 | 101 | /// Called by the frame work to save state before shutdown. 102 | fn save(&mut self, storage: &mut dyn eframe::Storage) { 103 | eframe::set_value(storage, eframe::APP_KEY, self); 104 | } 105 | 106 | fn clear_color(&self, _visuals: &egui::Visuals) -> [f32; 4] { 107 | self.settings.clear_color.to_normalized_gamma_f32() 108 | } 109 | } 110 | 111 | fn init_font(ctx: &Context) { 112 | let mut fonts = egui::FontDefinitions::default(); 113 | 114 | // Install my own font (maybe supporting non-latin characters). 115 | // .ttf and .otf files supported. 116 | fonts.font_data.insert( 117 | "my_font".to_owned(), 118 | egui::FontData::from_static(include_bytes!( 119 | "../../resources/fonts/NotoSansJP-Regular.ttf" 120 | )) 121 | .into(), 122 | ); 123 | 124 | // Put my font first (highest priority) for proportional text: 125 | fonts 126 | .families 127 | .entry(egui::FontFamily::Proportional) 128 | .or_default() 129 | .insert(0, "my_font".to_owned()); 130 | 131 | // Put my font as last fallback for monospace: 132 | fonts 133 | .families 134 | .entry(egui::FontFamily::Monospace) 135 | .or_default() 136 | .push("my_font".to_owned()); 137 | 138 | // Tell egui to use these fonts: 139 | ctx.set_fonts(fonts); 140 | } 141 | -------------------------------------------------------------------------------- /src/ui/background_rect.rs: -------------------------------------------------------------------------------- 1 | use super::{mouse_hover::get_frame_rect, screenshot_result_ui::scale_rect, settings::AppSettings}; 2 | use crate::action::{run_ocr, ScreenshotParameter, ScreenshotResult}; 3 | use crate::ocr::OcrBackend::MangaOcr; 4 | use crate::ui::event::Event::{UpdateBackendStatus, UpdateScreenshotResult}; 5 | use crate::ui::event::EventHandler; 6 | use crate::ui::settings::{Backend, BackendStatus}; 7 | use crate::ui::shutdown::TASK_TRACKER; 8 | use eframe::epaint::StrokeKind; 9 | use egui::{Color32, Context, Id, Pos2, Rect, Sense, TextureHandle, Vec2}; 10 | use log::{debug, warn}; 11 | use std::time::Duration; 12 | use tokio::time::Instant; 13 | 14 | #[derive(serde::Deserialize, serde::Serialize, Default)] 15 | #[serde(default)] 16 | pub struct BackgroundRect { 17 | start_pos: Pos2, 18 | end_pos: Pos2, 19 | 20 | pub screenshot_result: ScreenshotResult, 21 | #[serde(skip)] 22 | pub hide_ocr_rects: bool, 23 | 24 | #[serde(skip)] 25 | pub start_ocr_at: Option, 26 | #[serde(skip)] 27 | last_ocr_rect_hover_at: Option, 28 | 29 | #[serde(skip)] 30 | pub capture_image_handle: Option, 31 | #[serde(skip)] 32 | pub debug_image_handle: Option, 33 | } 34 | 35 | pub fn start_ocr_id() -> Id { 36 | Id::new("start_ocr") 37 | } 38 | 39 | fn is_start_ocr(ctx: &Context) -> bool { 40 | ctx.data_mut(|map| { 41 | let id = start_ocr_id(); 42 | let value = map.get_temp(id).unwrap_or(false); 43 | map.insert_temp(id, false); 44 | value 45 | }) 46 | } 47 | 48 | impl BackgroundRect { 49 | pub fn show(&mut self, ctx: &Context, settings: &AppSettings) { 50 | self.check_start_ocr(ctx, settings); 51 | 52 | let bg_response = self.draw_background(ctx); 53 | 54 | if !settings.mouse_passthrough && self.update_drag(&bg_response.response, ctx.zoom_factor()) 55 | { 56 | self.start_ocr_at = Some(Instant::now()); 57 | } 58 | 59 | if bg_response.response.drag_started() { 60 | self.screenshot_result = Default::default(); 61 | } 62 | 63 | if bg_response.response.dragged() { 64 | ctx.data_mut(|x| x.insert_temp(Id::new("ocr_is_cancelled"), true)); 65 | } 66 | 67 | if settings.show_capture_image { 68 | show_image_in_window(ctx, "Capture Image", self.capture_image_handle.clone()); 69 | } 70 | if settings.show_debug_image { 71 | show_image_in_window(ctx, "Debug Image", self.debug_image_handle.clone()); 72 | } 73 | } 74 | 75 | fn check_start_ocr(&mut self, ctx: &Context, settings: &AppSettings) { 76 | if self.hide_ocr_rects { 77 | //Rect are hidden => screenshot can be taken 78 | self.start_ocr(ctx, settings); 79 | self.hide_ocr_rects = false; 80 | } 81 | 82 | if is_start_ocr(ctx) || self.should_auto_restart(settings) { 83 | self.start_ocr_at = None; 84 | self.hide_ocr_rects = true; 85 | } 86 | } 87 | 88 | fn should_auto_restart(&mut self, settings: &AppSettings) -> bool { 89 | if let Some(instant) = self.start_ocr_at { 90 | let not_hovering = self.last_ocr_rect_hover_at.map_or(true, |x| { 91 | x.elapsed() >= Duration::from_millis(settings.hover_delay_ms) 92 | }); 93 | 94 | let elapsed = instant.elapsed(); 95 | return elapsed > Duration::from_millis(0) && not_hovering; 96 | } 97 | false 98 | } 99 | } 100 | 101 | fn show_image_in_window(ctx: &egui::Context, title: &str, texture: Option) { 102 | egui::Window::new(title).show(ctx, |ui| { 103 | if let Some(texture) = texture { 104 | ui.add( 105 | egui::Image::new(&texture) 106 | .shrink_to_fit() 107 | .corner_radius(10.0), 108 | ); 109 | } else { 110 | ui.label("No Image"); 111 | } 112 | }); 113 | } 114 | 115 | impl BackgroundRect { 116 | fn update_drag(&mut self, response: &egui::Response, zoom_factor: f32) -> bool { 117 | if response.drag_started() { 118 | if let Some(mpos) = response.interact_pointer_pos() { 119 | self.start_pos = mpos * zoom_factor; 120 | } 121 | } 122 | 123 | if response.dragged() { 124 | if let Some(mpos) = response.interact_pointer_pos() { 125 | self.end_pos = mpos * zoom_factor; 126 | } 127 | } 128 | 129 | if response.drag_stopped() { 130 | return true; 131 | } 132 | 133 | false 134 | } 135 | 136 | pub fn get_unscaled_rect(&self) -> Rect { 137 | Rect::from_two_pos(self.start_pos, self.end_pos) 138 | } 139 | 140 | pub fn get_global_rect(&self, ctx: &egui::Context) -> Rect { 141 | let mut rect = self.get_unscaled_rect(); 142 | let frame_rect = get_frame_rect(ctx); 143 | 144 | let zoom_factor = ctx.zoom_factor(); 145 | rect = rect.translate(Vec2::new( 146 | frame_rect.left() * zoom_factor, 147 | frame_rect.top() * zoom_factor, 148 | )); 149 | 150 | rect 151 | } 152 | 153 | fn start_ocr(&self, ctx: &egui::Context, settings: &AppSettings) { 154 | let global_rect = self.get_global_rect(ctx); 155 | 156 | let screenshot_parameter = ScreenshotParameter { 157 | x: global_rect.min.x as i32, 158 | y: global_rect.min.y as i32, 159 | width: global_rect.width() as u32, 160 | height: global_rect.height() as u32, 161 | detect_boxes: settings.detect_boxes, 162 | full_capture_ocr: !settings.detect_boxes, 163 | backends: vec![MangaOcr], 164 | threshold: settings.threshold, 165 | }; 166 | 167 | let Ok(image) = screenshot_parameter.get_screenshot() else { 168 | warn!("screenshot_parameter get screenshot failed"); 169 | return; 170 | }; 171 | 172 | ctx.data_mut(|x| x.insert_temp(Id::new("ocr_is_cancelled"), false)); 173 | 174 | let ctx = ctx.clone(); 175 | TASK_TRACKER.spawn(async move { 176 | debug!("Start ocr"); 177 | ctx.emit(UpdateBackendStatus( 178 | Backend::MangaOcr, 179 | BackendStatus::Running, 180 | )); 181 | let screenshot = run_ocr(screenshot_parameter, image).await.unwrap(); 182 | debug!("Start ocr done"); 183 | ctx.emit(UpdateBackendStatus(Backend::MangaOcr, BackendStatus::Ready)); 184 | 185 | ctx.emit(UpdateScreenshotResult(screenshot)); 186 | }); 187 | } 188 | 189 | fn draw_background(&mut self, ctx: &egui::Context) -> egui::InnerResponse<()> { 190 | let frame_rect = get_frame_rect(ctx); 191 | let rect = self.get_unscaled_rect(); 192 | 193 | let rect = scale_rect(rect, 1.0 / ctx.zoom_factor()); 194 | 195 | if !self.hide_ocr_rects && self.screenshot_result.show(ctx, &rect) { 196 | self.last_ocr_rect_hover_at = Some(Instant::now()); 197 | } 198 | 199 | egui::Area::new(Id::new("Background")) 200 | .order(egui::Order::Background) 201 | .sense(Sense::drag()) 202 | .fixed_pos(Pos2::ZERO) 203 | .show(ctx, |ui| { 204 | ui.set_width(frame_rect.width()); 205 | ui.set_height(frame_rect.height()); 206 | 207 | ui.painter().rect( 208 | rect, 209 | 0.0, 210 | Color32::TRANSPARENT, 211 | (1.0, Color32::RED), 212 | StrokeKind::Middle, 213 | ); 214 | }) 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /src/ui/event.rs: -------------------------------------------------------------------------------- 1 | use crate::action::ScreenshotResult; 2 | use crate::database::{HistoryData, KanjiStatistic}; 3 | use crate::jpn::JpnData; 4 | use crate::ui::settings::{Backend, BackendStatus}; 5 | use crate::OcrApp; 6 | use eframe::epaint::textures::TextureOptions; 7 | use eframe::epaint::ColorImage; 8 | use egui::{Context, Id, Memory, TextureHandle}; 9 | use image::DynamicImage; 10 | use std::ops::Add; 11 | use std::sync::LazyLock; 12 | use std::time::Duration; 13 | use tokio::time::Instant; 14 | 15 | #[derive(Debug, Clone)] 16 | pub enum Event { 17 | UpdateScreenshotResult(ScreenshotResult), 18 | UpdateHistoryData(Vec), 19 | UpdateKanjiStatistic(Vec), 20 | UpdateSelectedJpnData(JpnData), 21 | UpdateBackendStatus(Backend, BackendStatus), 22 | ResetUi, 23 | } 24 | 25 | pub trait EventHandler { 26 | fn emit(&self, value: Event); 27 | 28 | fn get_events(&self) -> Vec; 29 | 30 | fn update_state(&self, state: &mut OcrApp) { 31 | let events = self.get_events(); 32 | 33 | for x in events { 34 | self.handle_event(state, x); 35 | } 36 | } 37 | 38 | fn handle_event(&self, state: &mut OcrApp, event: Event); 39 | } 40 | 41 | static EVENT_LIST_ID: LazyLock = LazyLock::new(|| Id::new("EVENT_LIST")); 42 | 43 | impl EventHandler for Context { 44 | fn emit(&self, value: Event) { 45 | self.data_mut(|x| { 46 | x.get_temp_mut_or_insert_with(*EVENT_LIST_ID, Vec::new) 47 | .push(value); 48 | }); 49 | } 50 | 51 | fn get_events(&self) -> Vec { 52 | self.data_mut(|x| x.remove_temp(*EVENT_LIST_ID).unwrap_or_default()) 53 | } 54 | 55 | fn handle_event(&self, state: &mut OcrApp, event: Event) { 56 | match event { 57 | Event::UpdateScreenshotResult(result) => { 58 | if self 59 | .data(|x| x.get_temp(Id::new("ocr_is_cancelled"))) 60 | .unwrap_or(false) 61 | { 62 | return; 63 | } 64 | 65 | let background_rect = &mut state.background_rect; 66 | let settings = &state.settings; 67 | if settings.auto_restart_ocr { 68 | //Restart OCR 69 | background_rect.start_ocr_at = Some( 70 | Instant::now().add(Duration::from_millis(settings.auto_restart_delay_ms)), 71 | ); 72 | } 73 | 74 | background_rect.screenshot_result = result; 75 | 76 | background_rect.capture_image_handle = create_texture( 77 | self, 78 | background_rect.screenshot_result.capture_image.as_ref(), 79 | "capture_image_texture", 80 | ); 81 | 82 | background_rect.debug_image_handle = create_texture( 83 | self, 84 | background_rect.screenshot_result.debug_image.as_ref(), 85 | "debug_image_texture", 86 | ); 87 | } 88 | Event::UpdateHistoryData(data) => { 89 | state.history.history_data = data; 90 | } 91 | Event::UpdateKanjiStatistic(data) => { 92 | state.kanji_statistic.kanji_statistic = data; 93 | if state.kanji_statistic.selected_kanji_index.is_none() { 94 | state 95 | .kanji_statistic 96 | .update_selected_kanji_statistic(0, self); 97 | } 98 | } 99 | Event::UpdateSelectedJpnData(data) => { 100 | state.kanji_statistic.selected_jpn_data = data; 101 | } 102 | Event::UpdateBackendStatus(backend, status) => { 103 | backend.set_status(self, status); 104 | } 105 | Event::ResetUi => { 106 | self.memory_mut(|x| *x = Memory::default()); 107 | *state = OcrApp::default(); 108 | OcrApp::init_backends(self); 109 | } 110 | } 111 | } 112 | } 113 | 114 | fn create_texture( 115 | ctx: &Context, 116 | image: Option<&DynamicImage>, 117 | name: &str, 118 | ) -> Option { 119 | image.map(|image| { 120 | ctx.load_texture( 121 | name, 122 | ColorImage::from_rgba_unmultiplied( 123 | [image.width() as usize, image.height() as usize], 124 | image.clone().as_bytes(), 125 | ), 126 | TextureOptions::default(), 127 | ) 128 | }) 129 | } 130 | -------------------------------------------------------------------------------- /src/ui/kanji_history_ui.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use egui::{CentralPanel, Context, TopBottomPanel}; 4 | use egui_extras::{Column, TableBuilder}; 5 | use tokio::time::sleep; 6 | 7 | use crate::ui::event::Event::UpdateHistoryData; 8 | use crate::ui::event::EventHandler; 9 | use crate::ui::shutdown::TASK_TRACKER; 10 | use crate::{action, database::HistoryData}; 11 | 12 | #[derive(serde::Deserialize, serde::Serialize, Default)] 13 | #[serde(default)] 14 | pub struct HistoryDataUi { 15 | pub history_data: Vec, 16 | } 17 | pub fn init_history_updater(ctx: Context) { 18 | TASK_TRACKER.spawn(async move { 19 | loop { 20 | let history_data = action::load_history(); 21 | 22 | ctx.emit(UpdateHistoryData(history_data)); 23 | sleep(Duration::from_secs(1)).await; 24 | } 25 | }); 26 | } 27 | 28 | impl HistoryDataUi { 29 | pub fn show(&mut self, ctx: &egui::Context) { 30 | egui::Window::new("History").show(ctx, |ui| { 31 | TopBottomPanel::bottom("HistoryDataUi invisible bottom panel") 32 | .show_separator_line(false) 33 | .show_inside(ui, |_| ()); 34 | CentralPanel::default().show_inside(ui, |ui| self.show_table(ui)); 35 | }); 36 | } 37 | 38 | fn show_table(&mut self, ui: &mut egui::Ui) { 39 | TableBuilder::new(ui) 40 | .column(Column::auto()) 41 | .column(Column::remainder()) 42 | .column(Column::remainder()) 43 | .header(20.0, |mut header| { 44 | header.col(|ui| { 45 | ui.heading("Timestamp"); 46 | }); 47 | header.col(|ui| { 48 | ui.heading("OCR"); 49 | }); 50 | header.col(|ui| { 51 | ui.heading("Translation"); 52 | }); 53 | }) 54 | .body(|body| { 55 | body.rows(30.0, self.history_data.len(), |mut row| { 56 | if let Some(value) = self.history_data.get(row.index()) { 57 | row.col(|ui| { 58 | ui.label(&value.created_at); 59 | }); 60 | row.col(|ui| { 61 | ui.label(&value.ocr); 62 | }); 63 | row.col(|ui| { 64 | if let Some(translation) = &value.translation { 65 | ui.label(translation); 66 | } else if ui.button("Translate").clicked() { 67 | let ocr = value.ocr.clone(); 68 | TASK_TRACKER.spawn(async move { 69 | let _ = action::get_translation(&ocr).await; 70 | }); 71 | } 72 | }); 73 | } 74 | }); 75 | }); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/ui/kanji_statistic_ui.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use egui::{CentralPanel, Context, ScrollArea, Sense, SidePanel, TopBottomPanel}; 4 | use egui_extras::{Column, TableBuilder}; 5 | use tokio::time::sleep; 6 | 7 | use super::screenshot_result_ui::show_jpn_data_info; 8 | use crate::ui::event::Event::{UpdateKanjiStatistic, UpdateSelectedJpnData}; 9 | use crate::ui::event::EventHandler; 10 | use crate::ui::shutdown::TASK_TRACKER; 11 | use crate::{action, database::KanjiStatistic, jpn::JpnData}; 12 | 13 | #[derive(serde::Deserialize, serde::Serialize, Default)] 14 | #[serde(default)] 15 | pub struct KanjiStatisticUi { 16 | pub kanji_statistic: Vec, 17 | pub selected_kanji_index: Option, 18 | pub selected_jpn_data: JpnData, 19 | } 20 | 21 | pub fn init_kanji_statistic_updater(ctx: Context) { 22 | TASK_TRACKER.spawn(async move { 23 | loop { 24 | let kanji_statistic = action::load_statistic(); 25 | 26 | ctx.emit(UpdateKanjiStatistic(kanji_statistic)); 27 | sleep(Duration::from_secs(1)).await; 28 | } 29 | }); 30 | } 31 | 32 | impl KanjiStatisticUi { 33 | pub fn show(&mut self, ctx: &egui::Context) { 34 | egui::Window::new("Kanji Statistic").show(ctx, |ui| { 35 | SidePanel::left("Kanji Statistic Side Panel").show_inside(ui, |ui| { 36 | self.show_table(ui); 37 | }); 38 | TopBottomPanel::bottom("Kanji Statistic invisible bottom panel") 39 | .show_separator_line(false) 40 | .show_inside(ui, |_| ()); 41 | CentralPanel::default().show_inside(ui, |ui| { 42 | ScrollArea::vertical().show(ui, |ui| { 43 | ui.set_width(600.0); 44 | show_jpn_data_info(ui, &self.selected_jpn_data); 45 | }); 46 | }); 47 | }); 48 | } 49 | 50 | fn show_table(&mut self, ui: &mut egui::Ui) { 51 | let ctx = ui.ctx().clone(); 52 | TableBuilder::new(ui) 53 | .sense(Sense::click()) 54 | .column(Column::auto()) 55 | .column(Column::auto()) 56 | .header(20.0, |mut header| { 57 | header.col(|ui| { 58 | ui.heading("Kanji"); 59 | }); 60 | header.col(|ui| { 61 | ui.heading("Count"); 62 | }); 63 | }) 64 | .body(|body| { 65 | body.rows(30.0, self.kanji_statistic.len(), |mut row| { 66 | if let Some(value) = self.kanji_statistic.get(row.index()) { 67 | row.set_selected(self.selected_kanji_index == Some(row.index())); 68 | 69 | row.col(|ui| { 70 | ui.label(&value.kanji); 71 | }); 72 | row.col(|ui| { 73 | ui.label(format!("{}", &value.count)); 74 | }); 75 | 76 | if row.response().clicked() { 77 | self.update_selected_kanji_statistic(row.index(), &ctx); 78 | } 79 | } 80 | }); 81 | }); 82 | } 83 | 84 | pub(crate) fn update_selected_kanji_statistic(&mut self, index: usize, ctx: &Context) { 85 | self.selected_kanji_index = Some(index); 86 | if let Some(kanji_statistic) = self.kanji_statistic.get(index) { 87 | let kanji = kanji_statistic.kanji.clone(); 88 | let ctx = ctx.clone(); 89 | TASK_TRACKER.spawn(async move { 90 | if let Some(jpn_data) = action::get_kanji_jpn_data(&kanji).await { 91 | ctx.emit(UpdateSelectedJpnData(jpn_data)); 92 | }; 93 | }); 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/ui/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod app; 2 | pub mod background_rect; 3 | pub mod event; 4 | pub mod kanji_history_ui; 5 | pub mod kanji_statistic_ui; 6 | pub mod mouse_hover; 7 | pub mod screenshot_result_ui; 8 | pub mod settings; 9 | pub mod shutdown; 10 | -------------------------------------------------------------------------------- /src/ui/mouse_hover.rs: -------------------------------------------------------------------------------- 1 | use crate::OcrApp; 2 | use anyhow::Result; 3 | use egui::{Color32, Context, Id, Order, Pos2, Rect, Vec2}; 4 | use enigo::{Enigo, Mouse, Settings as EnigoSettings}; 5 | 6 | impl OcrApp { 7 | pub fn update_mouse_passthrough(&self, ctx: &Context) { 8 | ctx.send_viewport_cmd(egui::ViewportCommand::MousePassthrough( 9 | self.should_mouse_passthrough(ctx), 10 | )); 11 | } 12 | 13 | pub fn draw_mouse_position(&self, ctx: &Context) { 14 | let color = if self.should_mouse_passthrough(ctx) { 15 | Color32::RED 16 | } else { 17 | Color32::GREEN 18 | }; 19 | 20 | egui::Area::new(Id::new("Mouse Position")) 21 | .order(Order::Debug) 22 | .show(ctx, |ui| { 23 | if let Ok(position) = get_frame_mouse_position(ctx) { 24 | ui.painter().circle_filled(position, 10.0, color); 25 | } 26 | }); 27 | } 28 | 29 | fn should_mouse_passthrough(&self, ctx: &Context) -> bool { 30 | self.settings.mouse_passthrough && is_mouse_over_background(ctx) 31 | } 32 | } 33 | 34 | pub fn is_mouse_over_background(ctx: &Context) -> bool { 35 | let Ok(position) = get_frame_mouse_position(ctx) else { 36 | return false; 37 | }; 38 | if let Some(layer_id_at) = ctx.layer_id_at(position) { 39 | layer_id_at.order == Order::Background 40 | } else { 41 | false 42 | } 43 | } 44 | 45 | pub fn get_frame_mouse_position(ctx: &Context) -> Result { 46 | let frame_rect = get_frame_rect(ctx); 47 | 48 | let mouse_pos2 = get_mouse_position()?; 49 | 50 | let zoom_factor = ctx.zoom_factor(); 51 | let mouse_pos2 = Pos2::new(mouse_pos2.x / zoom_factor, mouse_pos2.y / zoom_factor); 52 | let Vec2 { x, y } = mouse_pos2 - frame_rect.min; 53 | 54 | Ok(Pos2::new(x, y)) 55 | } 56 | 57 | pub fn get_frame_rect(ctx: &Context) -> Rect { 58 | let mut frame_rect: Rect = Rect::ZERO; 59 | ctx.input(|x| { 60 | frame_rect = x.viewport().inner_rect.unwrap_or(Rect::ZERO); 61 | }); 62 | frame_rect 63 | } 64 | 65 | fn get_mouse_position() -> Result { 66 | let enigo = Enigo::new(&EnigoSettings::default())?; 67 | let (x, y) = enigo.location()?; 68 | 69 | Ok(Pos2::new(x as f32, y as f32)) 70 | } 71 | -------------------------------------------------------------------------------- /src/ui/screenshot_result_ui.rs: -------------------------------------------------------------------------------- 1 | use eframe::epaint::StrokeKind; 2 | use egui::{Align2, Color32, Id, Pos2, Rect, RichText, Sense, Vec2}; 3 | use std::time::{Duration, Instant}; 4 | 5 | use super::mouse_hover::get_frame_mouse_position; 6 | use crate::action::{self, get_translation, ResultData, ScreenshotResult}; 7 | use crate::ui::shutdown::TASK_TRACKER; 8 | 9 | impl ScreenshotResult { 10 | pub fn show(&mut self, ctx: &egui::Context, screenshot_rect: &Rect) -> bool { 11 | self.update_translation(ctx); 12 | 13 | let frame_mouse_position = get_frame_mouse_position(ctx).unwrap_or_default(); 14 | let mut area_hovered = false; 15 | 16 | let clicked_result_id = Id::new("clicked_result"); 17 | let clicked_result = ctx 18 | .data(|x| x.get_temp::(clicked_result_id)) 19 | .unwrap_or(-1); 20 | 21 | for (i, result) in self.ocr_results.iter().enumerate() { 22 | let rect = result.get_ui_rect(ctx); 23 | let rect = rect.translate(screenshot_rect.left_top().to_vec2()); 24 | let rect_is_clicked = clicked_result == i as i32; 25 | let area = egui::Area::new(Id::new(format!("ScreenshotResult {} {}", i, result.ocr))) 26 | .current_pos(rect.left_top()) 27 | .sense(Sense::click()) 28 | .show(ctx, |ui| { 29 | ui.set_width(rect.width()); 30 | ui.set_height(rect.height()); 31 | let contains = rect.contains(frame_mouse_position); 32 | 33 | let is_active = contains || rect_is_clicked; 34 | if is_active { 35 | show_ocr_info_window(ctx, &rect, result, i); 36 | } 37 | 38 | let color = if is_active { 39 | Color32::GREEN 40 | } else { 41 | Color32::BLUE 42 | }; 43 | 44 | ui.painter().rect( 45 | rect, 46 | 0.0, 47 | Color32::TRANSPARENT, 48 | (1.0, color), 49 | StrokeKind::Middle, 50 | ); 51 | }); 52 | 53 | if area.response.clicked() { 54 | if result.translation.is_empty() { 55 | fetch_translation(&result.ocr, i, ctx); 56 | } else { 57 | set_translation_visible(ctx, !is_translation_visible(ctx)); 58 | } 59 | } 60 | 61 | if area.response.secondary_clicked() { 62 | let value: i32 = if rect_is_clicked { -1 } else { i as i32 }; 63 | ctx.data_mut(|x| x.insert_temp(clicked_result_id, value)); 64 | } 65 | 66 | if area.response.hovered() { 67 | area_hovered = true; 68 | } 69 | } 70 | 71 | update_scroll_y_offset(ctx, area_hovered); 72 | area_hovered 73 | } 74 | 75 | fn update_translation(&mut self, ctx: &egui::Context) { 76 | let translation_id = Id::new("translation"); 77 | let update_translation = 78 | ctx.data_mut(|map| map.get_temp::(translation_id)); 79 | if let Some(update) = update_translation { 80 | self.ocr_results[update.index].translation = update.translation; 81 | ctx.data_mut(|x| x.remove_temp::(translation_id)); 82 | 83 | set_translation_visible(ctx, true); 84 | } 85 | } 86 | } 87 | 88 | fn is_translation_visible(ctx: &egui::Context) -> bool { 89 | ctx.data(|map| map.get_temp::(Id::new("is_translation_visible"))) 90 | .unwrap_or_default() 91 | } 92 | 93 | fn set_translation_visible(ctx: &egui::Context, is_visible: bool) { 94 | ctx.data_mut(|map| map.insert_temp::(Id::new("is_translation_visible"), is_visible)); 95 | } 96 | 97 | fn fetch_translation(ocr: &str, index: usize, ctx: &egui::Context) { 98 | let ocr = ocr.to_owned(); 99 | let ctx = ctx.clone(); 100 | tokio::spawn(async move { 101 | let translation = get_translation(&ocr).await; 102 | ctx.data_mut(|x| { 103 | x.insert_temp( 104 | Id::new("translation"), 105 | TranslationUpdate { index, translation }, 106 | ); 107 | }); 108 | }); 109 | } 110 | 111 | #[derive(Clone, Default)] 112 | struct TranslationUpdate { 113 | index: usize, 114 | translation: String, 115 | } 116 | 117 | fn update_scroll_y_offset(ctx: &egui::Context, area_hovered: bool) { 118 | let scroll_y_id = Id::new("Scroll Y"); 119 | 120 | // Reset the scroll offset when the area is hovered 121 | if is_area_hover_start(ctx, area_hovered) { 122 | ctx.data_mut(|map| map.insert_temp(scroll_y_id, 0)); 123 | } 124 | 125 | if !ctx.wants_pointer_input() { 126 | return; 127 | } 128 | 129 | let scroll_y = ctx.input(|state| state.raw_scroll_delta.y); 130 | if scroll_y == 0.0 { 131 | return; 132 | } 133 | 134 | let offset = if scroll_y > 0.0 { -1 } else { 1 }; 135 | ctx.data_mut(|map| { 136 | let value = map.get_temp::(scroll_y_id).unwrap_or_default() + offset; 137 | 138 | map.insert_temp(scroll_y_id, value); 139 | }); 140 | } 141 | 142 | fn is_area_hover_start(ctx: &egui::Context, area_hovered: bool) -> bool { 143 | let area_hovered_id = Id::new("area_hovered"); 144 | let old_area_hovered = ctx 145 | .data(|mem| mem.get_temp::(area_hovered_id)) 146 | .unwrap_or_default(); 147 | 148 | ctx.data_mut(|map| map.insert_temp(area_hovered_id, area_hovered)); 149 | !old_area_hovered && area_hovered 150 | } 151 | 152 | fn show_ocr_info_window(ctx: &egui::Context, rect: &Rect, result: &ResultData, index: usize) { 153 | let right_side = rect.min.x > ctx.screen_rect().width() * 2.0 / 3.0; 154 | 155 | let (pivot, default_pos_x) = if right_side { 156 | (Align2::RIGHT_TOP, rect.left() - 3.0) 157 | } else { 158 | (Align2::LEFT_TOP, rect.right() + 3.0) 159 | }; 160 | egui::Window::new(format!("OCR Info {} {}", index, result.ocr)) 161 | .title_bar(false) 162 | .pivot(pivot) 163 | .default_pos(Pos2::new(default_pos_x, rect.top())) 164 | .default_width(500.0) 165 | .show(ctx, |ui| { 166 | if !result.translation.is_empty() && is_translation_visible(ctx) { 167 | ui.label(get_info_text(&result.translation)); 168 | ui.separator(); 169 | } 170 | 171 | let id = Id::new("Scroll Y"); 172 | let index = ui.data(|map| map.get_temp(id)).unwrap_or_default(); 173 | let selected_jpn_data = result.get_jpn_data_with_info_by_index(index); 174 | for jpn in &result.jpn { 175 | ui.spacing_mut().item_spacing = Vec2::new(0.0, 0.0); 176 | ui.horizontal_wrapped(|ui| { 177 | for jpn_data in jpn { 178 | let kanji = jpn_data.get_kanji(); 179 | let mut text = get_info_text(&kanji); 180 | if jpn_data.has_kanji_data() { 181 | text = text.underline(); 182 | } 183 | if selected_jpn_data == Some(jpn_data) { 184 | text = text.color(Color32::RED); 185 | } 186 | ui.label(text); 187 | } 188 | }); 189 | } 190 | 191 | if let Some(info) = selected_jpn_data { 192 | ui.separator(); 193 | show_jpn_data_info(ui, info); 194 | update_kanji_statistic(ui, info); 195 | } 196 | }); 197 | } 198 | 199 | pub fn show_jpn_data_info(ui: &mut egui::Ui, info: &crate::jpn::JpnData) { 200 | for info_row in info.get_info_rows() { 201 | ui.label(get_info_text(info_row)); 202 | } 203 | } 204 | 205 | fn update_kanji_statistic(ui: &mut egui::Ui, info: &crate::jpn::JpnData) { 206 | let id = Id::new("show_kanji_timer"); 207 | let kanji_timer = ui.data(|x| x.get_temp::(id)); 208 | 209 | if let Some(mut timer) = kanji_timer { 210 | if !timer.statistic_updated && timer.timestamp.elapsed() >= Duration::from_millis(500) { 211 | timer.statistic_updated = true; 212 | ui.data_mut(|x| x.insert_temp(id, timer)); 213 | let kanji = info.get_kanji(); 214 | 215 | TASK_TRACKER.spawn(async move { 216 | let _ = action::increment_kanji_statistic(&kanji); 217 | }); 218 | return; 219 | } 220 | if timer.kanji == info.get_kanji() { 221 | return; 222 | } 223 | } 224 | ui.data_mut(|x| x.insert_temp(id, KanjiStatisticTimer::new(info.get_kanji()))); 225 | } 226 | 227 | #[derive(Clone, Debug)] 228 | struct KanjiStatisticTimer { 229 | kanji: String, 230 | timestamp: Instant, 231 | statistic_updated: bool, 232 | } 233 | 234 | impl KanjiStatisticTimer { 235 | fn new(kanji: String) -> Self { 236 | let timestamp = Instant::now(); 237 | Self { 238 | kanji, 239 | timestamp, 240 | statistic_updated: false, 241 | } 242 | } 243 | } 244 | 245 | fn get_info_text(text: impl Into) -> RichText { 246 | RichText::new(text).size(20.0) 247 | } 248 | 249 | impl ResultData { 250 | fn get_ui_rect(&self, ctx: &egui::Context) -> Rect { 251 | let zoom_factor = ctx.zoom_factor(); 252 | 253 | let rect = Rect::from_min_size( 254 | Pos2::new(self.x as f32, self.y as f32), 255 | Vec2 { 256 | x: self.w as f32, 257 | y: self.h as f32, 258 | }, 259 | ); 260 | scale_rect(rect, 1.0 / zoom_factor) 261 | } 262 | } 263 | 264 | pub fn scale_rect(rect: Rect, scale_factor: f32) -> Rect { 265 | Rect::from_min_size( 266 | Pos2::new(rect.min.x * scale_factor, rect.min.y * scale_factor), 267 | Vec2 { 268 | x: rect.width() * scale_factor, 269 | y: rect.height() * scale_factor, 270 | }, 271 | ) 272 | } 273 | -------------------------------------------------------------------------------- /src/ui/settings.rs: -------------------------------------------------------------------------------- 1 | use super::background_rect::start_ocr_id; 2 | use crate::action::open_workdir; 3 | use crate::ui::event::Event::ResetUi; 4 | use crate::ui::event::EventHandler; 5 | use egui::{Button, CollapsingHeader, Color32, Id, RichText, Spinner}; 6 | 7 | #[derive(serde::Deserialize, serde::Serialize)] 8 | #[serde(default)] 9 | pub struct AppSettings { 10 | pub clear_color: Color32, 11 | pub mouse_passthrough: bool, 12 | pub decorations: bool, 13 | pub zoom_factor: f32, 14 | 15 | pub auto_restart_ocr: bool, 16 | pub auto_restart_delay_ms: u64, 17 | pub hover_delay_ms: u64, 18 | 19 | //OCR Settings 20 | pub detect_boxes: bool, 21 | 22 | pub show_statistics: bool, 23 | pub show_history: bool, 24 | pub show_capture_image: bool, 25 | pub show_debug_image: bool, 26 | pub threshold: f32, 27 | 28 | pub show_debug_cursor: bool, 29 | } 30 | 31 | impl Default for AppSettings { 32 | fn default() -> Self { 33 | Self { 34 | clear_color: Color32::TRANSPARENT, 35 | mouse_passthrough: false, 36 | decorations: false, 37 | detect_boxes: true, 38 | zoom_factor: 1.5, 39 | auto_restart_ocr: true, 40 | auto_restart_delay_ms: 1000, 41 | hover_delay_ms: 1000, 42 | show_statistics: false, 43 | show_history: false, 44 | show_capture_image: false, 45 | show_debug_image: false, 46 | threshold: 0.5, 47 | show_debug_cursor: false, 48 | } 49 | } 50 | } 51 | 52 | impl AppSettings { 53 | pub(crate) fn show(&mut self, ctx: &egui::Context) { 54 | let window = egui::Window::new("Settings") 55 | .default_width(50.0) 56 | .resizable(false); 57 | window.show(ctx, |ui| { 58 | self.show_window_settings(ui); 59 | 60 | ui.horizontal(|ui| { 61 | Backend::MangaOcr.get_status_ui(ui); 62 | let enabled = Backend::MangaOcr.get_status(ui) == BackendStatus::Ready; 63 | if ui.add_enabled(enabled, Button::new("Start OCR")).clicked() { 64 | ui.data_mut(|map| map.insert_temp(start_ocr_id(), true)); 65 | } 66 | ui.checkbox(&mut self.auto_restart_ocr, "Auto Restart OCR"); 67 | }); 68 | 69 | self.show_ocr_config(ui); 70 | self.show_debug_config(ui); 71 | 72 | ui.separator(); 73 | ui.horizontal(|ui| { 74 | if ui.button(format!("{:^15}", "Quit")).clicked() { 75 | ctx.send_viewport_cmd(egui::ViewportCommand::Close); 76 | } 77 | ui.add_space(80.0); 78 | ui.hyperlink_to( 79 | "\u{E624} Manga Overlay on GitHub", 80 | "https://github.com/Icekey/manga-overlay", 81 | ); 82 | }); 83 | }); 84 | } 85 | 86 | fn show_window_settings(&mut self, ui: &mut egui::Ui) { 87 | egui::widgets::global_theme_preference_buttons(ui); 88 | 89 | ui.horizontal(|ui| { 90 | ui.label("Zoom Factor:"); 91 | ui.selectable_value(&mut self.zoom_factor, 1.0, "100%"); 92 | ui.selectable_value(&mut self.zoom_factor, 1.5, "150%"); 93 | ui.selectable_value(&mut self.zoom_factor, 2.0, "200%"); 94 | ui.selectable_value(&mut self.zoom_factor, 2.5, "250%"); 95 | ui.selectable_value(&mut self.zoom_factor, 3.0, "300%"); 96 | }); 97 | 98 | ui.checkbox(&mut self.mouse_passthrough, "Mouse Passthrough"); 99 | 100 | if ui.checkbox(&mut self.decorations, "Decorations").clicked() { 101 | ui.ctx() 102 | .send_viewport_cmd(egui::ViewportCommand::Decorations(self.decorations)); 103 | } 104 | 105 | ui.checkbox(&mut self.show_history, "Show History"); 106 | ui.checkbox(&mut self.show_statistics, "Show Statistics"); 107 | } 108 | 109 | fn show_ocr_config(&mut self, ui: &mut egui::Ui) { 110 | CollapsingHeader::new("OCR Config").show(ui, |ui| { 111 | ui.horizontal(|ui| { 112 | ui.selectable_value(&mut self.detect_boxes, false, "Full Capture"); 113 | ui.selectable_value(&mut self.detect_boxes, true, "Detect Boxes"); 114 | }); 115 | ui.horizontal(|ui| { 116 | if !self.detect_boxes { 117 | ui.disable() 118 | } 119 | ui.add(egui::Slider::new(&mut self.threshold, 0.0..=1.0).text("Box Threshold")); 120 | }); 121 | 122 | ui.horizontal(|ui| { 123 | ui.add( 124 | egui::Slider::new(&mut self.auto_restart_delay_ms, 0..=5000) 125 | .text("Auto Restart Time (ms)"), 126 | ); 127 | }); 128 | 129 | ui.horizontal(|ui| { 130 | ui.add( 131 | egui::Slider::new(&mut self.hover_delay_ms, 0..=5000).text("Hover Delay (ms)"), 132 | ); 133 | }); 134 | }); 135 | } 136 | 137 | fn show_debug_config(&mut self, ui: &mut egui::Ui) { 138 | CollapsingHeader::new("Debug Config").show(ui, |ui| { 139 | if ui.button("Open Workdir").clicked() { 140 | open_workdir(); 141 | } 142 | 143 | ui.horizontal(|ui| { 144 | ui.label("Background Color: "); 145 | ui.color_edit_button_srgba(&mut self.clear_color); 146 | }); 147 | 148 | ui.checkbox(&mut self.show_capture_image, "Show Capture Image"); 149 | ui.checkbox(&mut self.show_debug_image, "Show Debug Image"); 150 | ui.checkbox(&mut self.show_debug_cursor, "Show Debug Cursor"); 151 | 152 | if ui.button("Reset UI").clicked() { 153 | ui.ctx().emit(ResetUi); 154 | } 155 | }); 156 | } 157 | } 158 | 159 | #[derive(Debug, Clone, Eq, PartialEq)] 160 | pub enum BackendStatus { 161 | Loading, 162 | Ready, 163 | Running, 164 | Error, 165 | } 166 | 167 | impl BackendStatus { 168 | fn get_ui(&self, ui: &mut egui::Ui) { 169 | match self { 170 | BackendStatus::Loading => ui.add(Spinner::new()), 171 | BackendStatus::Ready | BackendStatus::Running => { 172 | ui.label(RichText::from("\u{2714}").color(Color32::GREEN)) 173 | } 174 | BackendStatus::Error => ui.label(RichText::from("\u{2716}").color(Color32::RED)), 175 | }; 176 | } 177 | } 178 | 179 | #[derive(Debug, Clone)] 180 | pub enum Backend { 181 | MangaOcr, 182 | } 183 | 184 | impl Backend { 185 | fn get_id(self: &Backend) -> Id { 186 | match self { 187 | Backend::MangaOcr => Id::new("MangaOcr_Status"), 188 | } 189 | } 190 | 191 | fn get_status(&self, ui: &egui::Ui) -> BackendStatus { 192 | ui.data(|data| { 193 | data.get_temp(self.get_id()) 194 | .unwrap_or_else(|| BackendStatus::Loading) 195 | }) 196 | } 197 | 198 | fn get_status_ui(&self, ui: &mut egui::Ui) { 199 | self.get_status(ui).get_ui(ui); 200 | } 201 | 202 | pub fn set_status(&self, ctx: &egui::Context, status: BackendStatus) { 203 | ctx.data_mut(|data| data.insert_temp(self.get_id(), status)); 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /src/ui/shutdown.rs: -------------------------------------------------------------------------------- 1 | use log::info; 2 | use std::sync::LazyLock; 3 | use tokio::spawn; 4 | use tokio_util::task::TaskTracker; 5 | 6 | pub static TASK_TRACKER: LazyLock = LazyLock::new(TaskTracker::new); 7 | 8 | pub fn shutdown_tasks() { 9 | let tracker = TASK_TRACKER.clone(); 10 | info!("start shutdown of {:?} tasks", tracker.len()); 11 | tracker.close(); 12 | 13 | spawn(async move { 14 | tracker.wait().await; 15 | info!("shutdown down"); 16 | }); 17 | } 18 | --------------------------------------------------------------------------------