├── .gitignore ├── LICENSE ├── Pipfile ├── Pipfile.lock ├── README.md ├── app.py ├── bys.mthli.com.conf ├── constants.py ├── database ├── chapter.py ├── data.py ├── feedback.py ├── sqlite.py ├── translation.py └── user.py ├── logger.py ├── openai.py ├── pm2.json ├── prompt.py ├── rds.py ├── sse.py ├── summary.py └── translation.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | # Others 163 | .DS_Store 164 | .vscode/ 165 | *.db 166 | *.db-shm 167 | *.db-wal 168 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU AFFERO GENERAL PUBLIC LICENSE 2 | Version 3, 19 November 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU Affero General Public License is a free, copyleft license for 11 | software and other kinds of works, specifically designed to ensure 12 | cooperation with the community in the case of network server software. 13 | 14 | The licenses for most software and other practical works are designed 15 | to take away your freedom to share and change the works. By contrast, 16 | our General Public Licenses are intended to guarantee your freedom to 17 | share and change all versions of a program--to make sure it remains free 18 | software for all its users. 19 | 20 | When we speak of free software, we are referring to freedom, not 21 | price. Our General Public Licenses are designed to make sure that you 22 | have the freedom to distribute copies of free software (and charge for 23 | them if you wish), that you receive source code or can get it if you 24 | want it, that you can change the software or use pieces of it in new 25 | free programs, and that you know you can do these things. 26 | 27 | Developers that use our General Public Licenses protect your rights 28 | with two steps: (1) assert copyright on the software, and (2) offer 29 | you this License which gives you legal permission to copy, distribute 30 | and/or modify the software. 31 | 32 | A secondary benefit of defending all users' freedom is that 33 | improvements made in alternate versions of the program, if they 34 | receive widespread use, become available for other developers to 35 | incorporate. Many developers of free software are heartened and 36 | encouraged by the resulting cooperation. However, in the case of 37 | software used on network servers, this result may fail to come about. 38 | The GNU General Public License permits making a modified version and 39 | letting the public access it on a server without ever releasing its 40 | source code to the public. 41 | 42 | The GNU Affero General Public License is designed specifically to 43 | ensure that, in such cases, the modified source code becomes available 44 | to the community. It requires the operator of a network server to 45 | provide the source code of the modified version running there to the 46 | users of that server. Therefore, public use of a modified version, on 47 | a publicly accessible server, gives the public access to the source 48 | code of the modified version. 49 | 50 | An older license, called the Affero General Public License and 51 | published by Affero, was designed to accomplish similar goals. This is 52 | a different license, not a version of the Affero GPL, but Affero has 53 | released a new version of the Affero GPL which permits relicensing under 54 | this license. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | TERMS AND CONDITIONS 60 | 61 | 0. Definitions. 62 | 63 | "This License" refers to version 3 of the GNU Affero General Public License. 64 | 65 | "Copyright" also means copyright-like laws that apply to other kinds of 66 | works, such as semiconductor masks. 67 | 68 | "The Program" refers to any copyrightable work licensed under this 69 | License. Each licensee is addressed as "you". "Licensees" and 70 | "recipients" may be individuals or organizations. 71 | 72 | To "modify" a work means to copy from or adapt all or part of the work 73 | in a fashion requiring copyright permission, other than the making of an 74 | exact copy. The resulting work is called a "modified version" of the 75 | earlier work or a work "based on" the earlier work. 76 | 77 | A "covered work" means either the unmodified Program or a work based 78 | on the Program. 79 | 80 | To "propagate" a work means to do anything with it that, without 81 | permission, would make you directly or secondarily liable for 82 | infringement under applicable copyright law, except executing it on a 83 | computer or modifying a private copy. Propagation includes copying, 84 | distribution (with or without modification), making available to the 85 | public, and in some countries other activities as well. 86 | 87 | To "convey" a work means any kind of propagation that enables other 88 | parties to make or receive copies. Mere interaction with a user through 89 | a computer network, with no transfer of a copy, is not conveying. 90 | 91 | An interactive user interface displays "Appropriate Legal Notices" 92 | to the extent that it includes a convenient and prominently visible 93 | feature that (1) displays an appropriate copyright notice, and (2) 94 | tells the user that there is no warranty for the work (except to the 95 | extent that warranties are provided), that licensees may convey the 96 | work under this License, and how to view a copy of this License. If 97 | the interface presents a list of user commands or options, such as a 98 | menu, a prominent item in the list meets this criterion. 99 | 100 | 1. Source Code. 101 | 102 | The "source code" for a work means the preferred form of the work 103 | for making modifications to it. "Object code" means any non-source 104 | form of a work. 105 | 106 | A "Standard Interface" means an interface that either is an official 107 | standard defined by a recognized standards body, or, in the case of 108 | interfaces specified for a particular programming language, one that 109 | is widely used among developers working in that language. 110 | 111 | The "System Libraries" of an executable work include anything, other 112 | than the work as a whole, that (a) is included in the normal form of 113 | packaging a Major Component, but which is not part of that Major 114 | Component, and (b) serves only to enable use of the work with that 115 | Major Component, or to implement a Standard Interface for which an 116 | implementation is available to the public in source code form. A 117 | "Major Component", in this context, means a major essential component 118 | (kernel, window system, and so on) of the specific operating system 119 | (if any) on which the executable work runs, or a compiler used to 120 | produce the work, or an object code interpreter used to run it. 121 | 122 | The "Corresponding Source" for a work in object code form means all 123 | the source code needed to generate, install, and (for an executable 124 | work) run the object code and to modify the work, including scripts to 125 | control those activities. However, it does not include the work's 126 | System Libraries, or general-purpose tools or generally available free 127 | programs which are used unmodified in performing those activities but 128 | which are not part of the work. For example, Corresponding Source 129 | includes interface definition files associated with source files for 130 | the work, and the source code for shared libraries and dynamically 131 | linked subprograms that the work is specifically designed to require, 132 | such as by intimate data communication or control flow between those 133 | subprograms and other parts of the work. 134 | 135 | The Corresponding Source need not include anything that users 136 | can regenerate automatically from other parts of the Corresponding 137 | Source. 138 | 139 | The Corresponding Source for a work in source code form is that 140 | same work. 141 | 142 | 2. Basic Permissions. 143 | 144 | All rights granted under this License are granted for the term of 145 | copyright on the Program, and are irrevocable provided the stated 146 | conditions are met. This License explicitly affirms your unlimited 147 | permission to run the unmodified Program. The output from running a 148 | covered work is covered by this License only if the output, given its 149 | content, constitutes a covered work. This License acknowledges your 150 | rights of fair use or other equivalent, as provided by copyright law. 151 | 152 | You may make, run and propagate covered works that you do not 153 | convey, without conditions so long as your license otherwise remains 154 | in force. You may convey covered works to others for the sole purpose 155 | of having them make modifications exclusively for you, or provide you 156 | with facilities for running those works, provided that you comply with 157 | the terms of this License in conveying all material for which you do 158 | not control copyright. Those thus making or running the covered works 159 | for you must do so exclusively on your behalf, under your direction 160 | and control, on terms that prohibit them from making any copies of 161 | your copyrighted material outside their relationship with you. 162 | 163 | Conveying under any other circumstances is permitted solely under 164 | the conditions stated below. Sublicensing is not allowed; section 10 165 | makes it unnecessary. 166 | 167 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 | 169 | No covered work shall be deemed part of an effective technological 170 | measure under any applicable law fulfilling obligations under article 171 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 | similar laws prohibiting or restricting circumvention of such 173 | measures. 174 | 175 | When you convey a covered work, you waive any legal power to forbid 176 | circumvention of technological measures to the extent such circumvention 177 | is effected by exercising rights under this License with respect to 178 | the covered work, and you disclaim any intention to limit operation or 179 | modification of the work as a means of enforcing, against the work's 180 | users, your or third parties' legal rights to forbid circumvention of 181 | technological measures. 182 | 183 | 4. Conveying Verbatim Copies. 184 | 185 | You may convey verbatim copies of the Program's source code as you 186 | receive it, in any medium, provided that you conspicuously and 187 | appropriately publish on each copy an appropriate copyright notice; 188 | keep intact all notices stating that this License and any 189 | non-permissive terms added in accord with section 7 apply to the code; 190 | keep intact all notices of the absence of any warranty; and give all 191 | recipients a copy of this License along with the Program. 192 | 193 | You may charge any price or no price for each copy that you convey, 194 | and you may offer support or warranty protection for a fee. 195 | 196 | 5. Conveying Modified Source Versions. 197 | 198 | You may convey a work based on the Program, or the modifications to 199 | produce it from the Program, in the form of source code under the 200 | terms of section 4, provided that you also meet all of these conditions: 201 | 202 | a) The work must carry prominent notices stating that you modified 203 | it, and giving a relevant date. 204 | 205 | b) The work must carry prominent notices stating that it is 206 | released under this License and any conditions added under section 207 | 7. This requirement modifies the requirement in section 4 to 208 | "keep intact all notices". 209 | 210 | c) You must license the entire work, as a whole, under this 211 | License to anyone who comes into possession of a copy. This 212 | License will therefore apply, along with any applicable section 7 213 | additional terms, to the whole of the work, and all its parts, 214 | regardless of how they are packaged. This License gives no 215 | permission to license the work in any other way, but it does not 216 | invalidate such permission if you have separately received it. 217 | 218 | d) If the work has interactive user interfaces, each must display 219 | Appropriate Legal Notices; however, if the Program has interactive 220 | interfaces that do not display Appropriate Legal Notices, your 221 | work need not make them do so. 222 | 223 | A compilation of a covered work with other separate and independent 224 | works, which are not by their nature extensions of the covered work, 225 | and which are not combined with it such as to form a larger program, 226 | in or on a volume of a storage or distribution medium, is called an 227 | "aggregate" if the compilation and its resulting copyright are not 228 | used to limit the access or legal rights of the compilation's users 229 | beyond what the individual works permit. Inclusion of a covered work 230 | in an aggregate does not cause this License to apply to the other 231 | parts of the aggregate. 232 | 233 | 6. Conveying Non-Source Forms. 234 | 235 | You may convey a covered work in object code form under the terms 236 | of sections 4 and 5, provided that you also convey the 237 | machine-readable Corresponding Source under the terms of this License, 238 | in one of these ways: 239 | 240 | a) Convey the object code in, or embodied in, a physical product 241 | (including a physical distribution medium), accompanied by the 242 | Corresponding Source fixed on a durable physical medium 243 | customarily used for software interchange. 244 | 245 | b) Convey the object code in, or embodied in, a physical product 246 | (including a physical distribution medium), accompanied by a 247 | written offer, valid for at least three years and valid for as 248 | long as you offer spare parts or customer support for that product 249 | model, to give anyone who possesses the object code either (1) a 250 | copy of the Corresponding Source for all the software in the 251 | product that is covered by this License, on a durable physical 252 | medium customarily used for software interchange, for a price no 253 | more than your reasonable cost of physically performing this 254 | conveying of source, or (2) access to copy the 255 | Corresponding Source from a network server at no charge. 256 | 257 | c) Convey individual copies of the object code with a copy of the 258 | written offer to provide the Corresponding Source. This 259 | alternative is allowed only occasionally and noncommercially, and 260 | only if you received the object code with such an offer, in accord 261 | with subsection 6b. 262 | 263 | d) Convey the object code by offering access from a designated 264 | place (gratis or for a charge), and offer equivalent access to the 265 | Corresponding Source in the same way through the same place at no 266 | further charge. You need not require recipients to copy the 267 | Corresponding Source along with the object code. If the place to 268 | copy the object code is a network server, the Corresponding Source 269 | may be on a different server (operated by you or a third party) 270 | that supports equivalent copying facilities, provided you maintain 271 | clear directions next to the object code saying where to find the 272 | Corresponding Source. Regardless of what server hosts the 273 | Corresponding Source, you remain obligated to ensure that it is 274 | available for as long as needed to satisfy these requirements. 275 | 276 | e) Convey the object code using peer-to-peer transmission, provided 277 | you inform other peers where the object code and Corresponding 278 | Source of the work are being offered to the general public at no 279 | charge under subsection 6d. 280 | 281 | A separable portion of the object code, whose source code is excluded 282 | from the Corresponding Source as a System Library, need not be 283 | included in conveying the object code work. 284 | 285 | A "User Product" is either (1) a "consumer product", which means any 286 | tangible personal property which is normally used for personal, family, 287 | or household purposes, or (2) anything designed or sold for incorporation 288 | into a dwelling. In determining whether a product is a consumer product, 289 | doubtful cases shall be resolved in favor of coverage. For a particular 290 | product received by a particular user, "normally used" refers to a 291 | typical or common use of that class of product, regardless of the status 292 | of the particular user or of the way in which the particular user 293 | actually uses, or expects or is expected to use, the product. A product 294 | is a consumer product regardless of whether the product has substantial 295 | commercial, industrial or non-consumer uses, unless such uses represent 296 | the only significant mode of use of the product. 297 | 298 | "Installation Information" for a User Product means any methods, 299 | procedures, authorization keys, or other information required to install 300 | and execute modified versions of a covered work in that User Product from 301 | a modified version of its Corresponding Source. The information must 302 | suffice to ensure that the continued functioning of the modified object 303 | code is in no case prevented or interfered with solely because 304 | modification has been made. 305 | 306 | If you convey an object code work under this section in, or with, or 307 | specifically for use in, a User Product, and the conveying occurs as 308 | part of a transaction in which the right of possession and use of the 309 | User Product is transferred to the recipient in perpetuity or for a 310 | fixed term (regardless of how the transaction is characterized), the 311 | Corresponding Source conveyed under this section must be accompanied 312 | by the Installation Information. But this requirement does not apply 313 | if neither you nor any third party retains the ability to install 314 | modified object code on the User Product (for example, the work has 315 | been installed in ROM). 316 | 317 | The requirement to provide Installation Information does not include a 318 | requirement to continue to provide support service, warranty, or updates 319 | for a work that has been modified or installed by the recipient, or for 320 | the User Product in which it has been modified or installed. Access to a 321 | network may be denied when the modification itself materially and 322 | adversely affects the operation of the network or violates the rules and 323 | protocols for communication across the network. 324 | 325 | Corresponding Source conveyed, and Installation Information provided, 326 | in accord with this section must be in a format that is publicly 327 | documented (and with an implementation available to the public in 328 | source code form), and must require no special password or key for 329 | unpacking, reading or copying. 330 | 331 | 7. Additional Terms. 332 | 333 | "Additional permissions" are terms that supplement the terms of this 334 | License by making exceptions from one or more of its conditions. 335 | Additional permissions that are applicable to the entire Program shall 336 | be treated as though they were included in this License, to the extent 337 | that they are valid under applicable law. If additional permissions 338 | apply only to part of the Program, that part may be used separately 339 | under those permissions, but the entire Program remains governed by 340 | this License without regard to the additional permissions. 341 | 342 | When you convey a copy of a covered work, you may at your option 343 | remove any additional permissions from that copy, or from any part of 344 | it. (Additional permissions may be written to require their own 345 | removal in certain cases when you modify the work.) You may place 346 | additional permissions on material, added by you to a covered work, 347 | for which you have or can give appropriate copyright permission. 348 | 349 | Notwithstanding any other provision of this License, for material you 350 | add to a covered work, you may (if authorized by the copyright holders of 351 | that material) supplement the terms of this License with terms: 352 | 353 | a) Disclaiming warranty or limiting liability differently from the 354 | terms of sections 15 and 16 of this License; or 355 | 356 | b) Requiring preservation of specified reasonable legal notices or 357 | author attributions in that material or in the Appropriate Legal 358 | Notices displayed by works containing it; or 359 | 360 | c) Prohibiting misrepresentation of the origin of that material, or 361 | requiring that modified versions of such material be marked in 362 | reasonable ways as different from the original version; or 363 | 364 | d) Limiting the use for publicity purposes of names of licensors or 365 | authors of the material; or 366 | 367 | e) Declining to grant rights under trademark law for use of some 368 | trade names, trademarks, or service marks; or 369 | 370 | f) Requiring indemnification of licensors and authors of that 371 | material by anyone who conveys the material (or modified versions of 372 | it) with contractual assumptions of liability to the recipient, for 373 | any liability that these contractual assumptions directly impose on 374 | those licensors and authors. 375 | 376 | All other non-permissive additional terms are considered "further 377 | restrictions" within the meaning of section 10. If the Program as you 378 | received it, or any part of it, contains a notice stating that it is 379 | governed by this License along with a term that is a further 380 | restriction, you may remove that term. If a license document contains 381 | a further restriction but permits relicensing or conveying under this 382 | License, you may add to a covered work material governed by the terms 383 | of that license document, provided that the further restriction does 384 | not survive such relicensing or conveying. 385 | 386 | If you add terms to a covered work in accord with this section, you 387 | must place, in the relevant source files, a statement of the 388 | additional terms that apply to those files, or a notice indicating 389 | where to find the applicable terms. 390 | 391 | Additional terms, permissive or non-permissive, may be stated in the 392 | form of a separately written license, or stated as exceptions; 393 | the above requirements apply either way. 394 | 395 | 8. Termination. 396 | 397 | You may not propagate or modify a covered work except as expressly 398 | provided under this License. Any attempt otherwise to propagate or 399 | modify it is void, and will automatically terminate your rights under 400 | this License (including any patent licenses granted under the third 401 | paragraph of section 11). 402 | 403 | However, if you cease all violation of this License, then your 404 | license from a particular copyright holder is reinstated (a) 405 | provisionally, unless and until the copyright holder explicitly and 406 | finally terminates your license, and (b) permanently, if the copyright 407 | holder fails to notify you of the violation by some reasonable means 408 | prior to 60 days after the cessation. 409 | 410 | Moreover, your license from a particular copyright holder is 411 | reinstated permanently if the copyright holder notifies you of the 412 | violation by some reasonable means, this is the first time you have 413 | received notice of violation of this License (for any work) from that 414 | copyright holder, and you cure the violation prior to 30 days after 415 | your receipt of the notice. 416 | 417 | Termination of your rights under this section does not terminate the 418 | licenses of parties who have received copies or rights from you under 419 | this License. If your rights have been terminated and not permanently 420 | reinstated, you do not qualify to receive new licenses for the same 421 | material under section 10. 422 | 423 | 9. Acceptance Not Required for Having Copies. 424 | 425 | You are not required to accept this License in order to receive or 426 | run a copy of the Program. Ancillary propagation of a covered work 427 | occurring solely as a consequence of using peer-to-peer transmission 428 | to receive a copy likewise does not require acceptance. However, 429 | nothing other than this License grants you permission to propagate or 430 | modify any covered work. These actions infringe copyright if you do 431 | not accept this License. Therefore, by modifying or propagating a 432 | covered work, you indicate your acceptance of this License to do so. 433 | 434 | 10. Automatic Licensing of Downstream Recipients. 435 | 436 | Each time you convey a covered work, the recipient automatically 437 | receives a license from the original licensors, to run, modify and 438 | propagate that work, subject to this License. You are not responsible 439 | for enforcing compliance by third parties with this License. 440 | 441 | An "entity transaction" is a transaction transferring control of an 442 | organization, or substantially all assets of one, or subdividing an 443 | organization, or merging organizations. If propagation of a covered 444 | work results from an entity transaction, each party to that 445 | transaction who receives a copy of the work also receives whatever 446 | licenses to the work the party's predecessor in interest had or could 447 | give under the previous paragraph, plus a right to possession of the 448 | Corresponding Source of the work from the predecessor in interest, if 449 | the predecessor has it or can get it with reasonable efforts. 450 | 451 | You may not impose any further restrictions on the exercise of the 452 | rights granted or affirmed under this License. For example, you may 453 | not impose a license fee, royalty, or other charge for exercise of 454 | rights granted under this License, and you may not initiate litigation 455 | (including a cross-claim or counterclaim in a lawsuit) alleging that 456 | any patent claim is infringed by making, using, selling, offering for 457 | sale, or importing the Program or any portion of it. 458 | 459 | 11. Patents. 460 | 461 | A "contributor" is a copyright holder who authorizes use under this 462 | License of the Program or a work on which the Program is based. The 463 | work thus licensed is called the contributor's "contributor version". 464 | 465 | A contributor's "essential patent claims" are all patent claims 466 | owned or controlled by the contributor, whether already acquired or 467 | hereafter acquired, that would be infringed by some manner, permitted 468 | by this License, of making, using, or selling its contributor version, 469 | but do not include claims that would be infringed only as a 470 | consequence of further modification of the contributor version. For 471 | purposes of this definition, "control" includes the right to grant 472 | patent sublicenses in a manner consistent with the requirements of 473 | this License. 474 | 475 | Each contributor grants you a non-exclusive, worldwide, royalty-free 476 | patent license under the contributor's essential patent claims, to 477 | make, use, sell, offer for sale, import and otherwise run, modify and 478 | propagate the contents of its contributor version. 479 | 480 | In the following three paragraphs, a "patent license" is any express 481 | agreement or commitment, however denominated, not to enforce a patent 482 | (such as an express permission to practice a patent or covenant not to 483 | sue for patent infringement). To "grant" such a patent license to a 484 | party means to make such an agreement or commitment not to enforce a 485 | patent against the party. 486 | 487 | If you convey a covered work, knowingly relying on a patent license, 488 | and the Corresponding Source of the work is not available for anyone 489 | to copy, free of charge and under the terms of this License, through a 490 | publicly available network server or other readily accessible means, 491 | then you must either (1) cause the Corresponding Source to be so 492 | available, or (2) arrange to deprive yourself of the benefit of the 493 | patent license for this particular work, or (3) arrange, in a manner 494 | consistent with the requirements of this License, to extend the patent 495 | license to downstream recipients. "Knowingly relying" means you have 496 | actual knowledge that, but for the patent license, your conveying the 497 | covered work in a country, or your recipient's use of the covered work 498 | in a country, would infringe one or more identifiable patents in that 499 | country that you have reason to believe are valid. 500 | 501 | If, pursuant to or in connection with a single transaction or 502 | arrangement, you convey, or propagate by procuring conveyance of, a 503 | covered work, and grant a patent license to some of the parties 504 | receiving the covered work authorizing them to use, propagate, modify 505 | or convey a specific copy of the covered work, then the patent license 506 | you grant is automatically extended to all recipients of the covered 507 | work and works based on it. 508 | 509 | A patent license is "discriminatory" if it does not include within 510 | the scope of its coverage, prohibits the exercise of, or is 511 | conditioned on the non-exercise of one or more of the rights that are 512 | specifically granted under this License. You may not convey a covered 513 | work if you are a party to an arrangement with a third party that is 514 | in the business of distributing software, under which you make payment 515 | to the third party based on the extent of your activity of conveying 516 | the work, and under which the third party grants, to any of the 517 | parties who would receive the covered work from you, a discriminatory 518 | patent license (a) in connection with copies of the covered work 519 | conveyed by you (or copies made from those copies), or (b) primarily 520 | for and in connection with specific products or compilations that 521 | contain the covered work, unless you entered into that arrangement, 522 | or that patent license was granted, prior to 28 March 2007. 523 | 524 | Nothing in this License shall be construed as excluding or limiting 525 | any implied license or other defenses to infringement that may 526 | otherwise be available to you under applicable patent law. 527 | 528 | 12. No Surrender of Others' Freedom. 529 | 530 | If conditions are imposed on you (whether by court order, agreement or 531 | otherwise) that contradict the conditions of this License, they do not 532 | excuse you from the conditions of this License. If you cannot convey a 533 | covered work so as to satisfy simultaneously your obligations under this 534 | License and any other pertinent obligations, then as a consequence you may 535 | not convey it at all. For example, if you agree to terms that obligate you 536 | to collect a royalty for further conveying from those to whom you convey 537 | the Program, the only way you could satisfy both those terms and this 538 | License would be to refrain entirely from conveying the Program. 539 | 540 | 13. Remote Network Interaction; Use with the GNU General Public License. 541 | 542 | Notwithstanding any other provision of this License, if you modify the 543 | Program, your modified version must prominently offer all users 544 | interacting with it remotely through a computer network (if your version 545 | supports such interaction) an opportunity to receive the Corresponding 546 | Source of your version by providing access to the Corresponding Source 547 | from a network server at no charge, through some standard or customary 548 | means of facilitating copying of software. This Corresponding Source 549 | shall include the Corresponding Source for any work covered by version 3 550 | of the GNU General Public License that is incorporated pursuant to the 551 | following paragraph. 552 | 553 | Notwithstanding any other provision of this License, you have 554 | permission to link or combine any covered work with a work licensed 555 | under version 3 of the GNU General Public License into a single 556 | combined work, and to convey the resulting work. The terms of this 557 | License will continue to apply to the part which is the covered work, 558 | but the work with which it is combined will remain governed by version 559 | 3 of the GNU General Public License. 560 | 561 | 14. Revised Versions of this License. 562 | 563 | The Free Software Foundation may publish revised and/or new versions of 564 | the GNU Affero General Public License from time to time. Such new versions 565 | will be similar in spirit to the present version, but may differ in detail to 566 | address new problems or concerns. 567 | 568 | Each version is given a distinguishing version number. If the 569 | Program specifies that a certain numbered version of the GNU Affero General 570 | Public License "or any later version" applies to it, you have the 571 | option of following the terms and conditions either of that numbered 572 | version or of any later version published by the Free Software 573 | Foundation. If the Program does not specify a version number of the 574 | GNU Affero General Public License, you may choose any version ever published 575 | by the Free Software Foundation. 576 | 577 | If the Program specifies that a proxy can decide which future 578 | versions of the GNU Affero General Public License can be used, that proxy's 579 | public statement of acceptance of a version permanently authorizes you 580 | to choose that version for the Program. 581 | 582 | Later license versions may give you additional or different 583 | permissions. However, no additional obligations are imposed on any 584 | author or copyright holder as a result of your choosing to follow a 585 | later version. 586 | 587 | 15. Disclaimer of Warranty. 588 | 589 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 | 598 | 16. Limitation of Liability. 599 | 600 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 | SUCH DAMAGES. 609 | 610 | 17. Interpretation of Sections 15 and 16. 611 | 612 | If the disclaimer of warranty and limitation of liability provided 613 | above cannot be given local legal effect according to their terms, 614 | reviewing courts shall apply local law that most closely approximates 615 | an absolute waiver of all civil liability in connection with the 616 | Program, unless a warranty or assumption of liability accompanies a 617 | copy of the Program in return for a fee. 618 | 619 | END OF TERMS AND CONDITIONS 620 | 621 | How to Apply These Terms to Your New Programs 622 | 623 | If you develop a new program, and you want it to be of the greatest 624 | possible use to the public, the best way to achieve this is to make it 625 | free software which everyone can redistribute and change under these terms. 626 | 627 | To do so, attach the following notices to the program. It is safest 628 | to attach them to the start of each source file to most effectively 629 | state the exclusion of warranty; and each file should have at least 630 | the "copyright" line and a pointer to where the full notice is found. 631 | 632 | 633 | Copyright (C) 634 | 635 | This program is free software: you can redistribute it and/or modify 636 | it under the terms of the GNU Affero General Public License as published 637 | by the Free Software Foundation, either version 3 of the License, or 638 | (at your option) any later version. 639 | 640 | This program is distributed in the hope that it will be useful, 641 | but WITHOUT ANY WARRANTY; without even the implied warranty of 642 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 | GNU Affero General Public License for more details. 644 | 645 | You should have received a copy of the GNU Affero General Public License 646 | along with this program. If not, see . 647 | 648 | Also add information on how to contact you by electronic and paper mail. 649 | 650 | If your software can interact with users remotely through a computer 651 | network, you should also make sure that it provides a way for users to 652 | get its source. For example, if your program is a web application, its 653 | interface could display a "Source" link that leads users to an archive 654 | of the code. There are many ways you could offer source, and different 655 | solutions will be better for different programs; see section 13 for the 656 | specific requirements. 657 | 658 | You should also get your employer (if you work as a programmer) or school, 659 | if any, to sign a "copyright disclaimer" for the program, if necessary. 660 | For more information on this, and how to apply and follow the GNU AGPL, see 661 | . 662 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | redis = "*" 8 | tiktoken = "*" 9 | ipython = "*" 10 | httpx = "*" 11 | strenum = "*" 12 | tenacity = "*" 13 | langcodes = "*" 14 | language-data = "*" 15 | quart = "*" 16 | hypercorn = "*" 17 | arq = "*" 18 | youtube-transcript-api = "*" 19 | quart-cors = "*" 20 | 21 | [dev-packages] 22 | autopep8 = "*" 23 | 24 | [requires] 25 | python_version = "3.9" 26 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "0984ce84f00ab993889dd2dd12760f68d6063c3e7d69a6e1c65d2c54f0da3a1e" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.9" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "aiofiles": { 20 | "hashes": [ 21 | "sha256:9312414ae06472eb6f1d163f555e466a23aed1c8f60c30cccf7121dba2e53eb2", 22 | "sha256:edd247df9a19e0db16534d4baaf536d6609a43e1de5401d7a4c1c148753a1635" 23 | ], 24 | "markers": "python_version >= '3.7' and python_version < '4.0'", 25 | "version": "==23.1.0" 26 | }, 27 | "anyio": { 28 | "hashes": [ 29 | "sha256:275d9973793619a5374e1c89a4f4ad3f4b0a5510a2b5b939444bee8f4c4d37ce", 30 | "sha256:eddca883c4175f14df8aedce21054bfca3adb70ffe76a9f607aef9d7fa2ea7f0" 31 | ], 32 | "markers": "python_version >= '3.7'", 33 | "version": "==3.7.0" 34 | }, 35 | "appnope": { 36 | "hashes": [ 37 | "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24", 38 | "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e" 39 | ], 40 | "markers": "sys_platform == 'darwin'", 41 | "version": "==0.1.3" 42 | }, 43 | "arq": { 44 | "hashes": [ 45 | "sha256:d176ebadfba920c039dc578814d19b7814d67fa15f82fdccccaedb4330d65dae", 46 | "sha256:db072d0f39c0bc06b436db67ae1f315c81abc1527563b828955670531815290b" 47 | ], 48 | "index": "pypi", 49 | "version": "==0.25.0" 50 | }, 51 | "asttokens": { 52 | "hashes": [ 53 | "sha256:4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3", 54 | "sha256:6b0ac9e93fb0335014d382b8fa9b3afa7df546984258005da0b9e7095b3deb1c" 55 | ], 56 | "version": "==2.2.1" 57 | }, 58 | "async-timeout": { 59 | "hashes": [ 60 | "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15", 61 | "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c" 62 | ], 63 | "markers": "python_full_version <= '3.11.2'", 64 | "version": "==4.0.2" 65 | }, 66 | "backcall": { 67 | "hashes": [ 68 | "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e", 69 | "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255" 70 | ], 71 | "version": "==0.2.0" 72 | }, 73 | "blinker": { 74 | "hashes": [ 75 | "sha256:1eb563df6fdbc39eeddc177d953203f99f097e9bf0e2b8f9f3cf18b6ca425e36", 76 | "sha256:923e5e2f69c155f2cc42dafbbd70e16e3fde24d2d4aa2ab72fbe386238892462" 77 | ], 78 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", 79 | "version": "==1.5" 80 | }, 81 | "certifi": { 82 | "hashes": [ 83 | "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7", 84 | "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716" 85 | ], 86 | "markers": "python_version >= '3.6'", 87 | "version": "==2023.5.7" 88 | }, 89 | "charset-normalizer": { 90 | "hashes": [ 91 | "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6", 92 | "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1", 93 | "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e", 94 | "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373", 95 | "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62", 96 | "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230", 97 | "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be", 98 | "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c", 99 | "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0", 100 | "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448", 101 | "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f", 102 | "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649", 103 | "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d", 104 | "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0", 105 | "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706", 106 | "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a", 107 | "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59", 108 | "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23", 109 | "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5", 110 | "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb", 111 | "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e", 112 | "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e", 113 | "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c", 114 | "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28", 115 | "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d", 116 | "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41", 117 | "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974", 118 | "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce", 119 | "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f", 120 | "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1", 121 | "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d", 122 | "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8", 123 | "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017", 124 | "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31", 125 | "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7", 126 | "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8", 127 | "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e", 128 | "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14", 129 | "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd", 130 | "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d", 131 | "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795", 132 | "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b", 133 | "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b", 134 | "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b", 135 | "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203", 136 | "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f", 137 | "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19", 138 | "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1", 139 | "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a", 140 | "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac", 141 | "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9", 142 | "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0", 143 | "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137", 144 | "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f", 145 | "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6", 146 | "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5", 147 | "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909", 148 | "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f", 149 | "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0", 150 | "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324", 151 | "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755", 152 | "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb", 153 | "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854", 154 | "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c", 155 | "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60", 156 | "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84", 157 | "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0", 158 | "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b", 159 | "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1", 160 | "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531", 161 | "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1", 162 | "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11", 163 | "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326", 164 | "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df", 165 | "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab" 166 | ], 167 | "markers": "python_full_version >= '3.7.0'", 168 | "version": "==3.1.0" 169 | }, 170 | "click": { 171 | "hashes": [ 172 | "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", 173 | "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" 174 | ], 175 | "markers": "python_version >= '3.7'", 176 | "version": "==8.1.3" 177 | }, 178 | "decorator": { 179 | "hashes": [ 180 | "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330", 181 | "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186" 182 | ], 183 | "markers": "python_version >= '3.5'", 184 | "version": "==5.1.1" 185 | }, 186 | "exceptiongroup": { 187 | "hashes": [ 188 | "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e", 189 | "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785" 190 | ], 191 | "markers": "python_version < '3.11'", 192 | "version": "==1.1.1" 193 | }, 194 | "executing": { 195 | "hashes": [ 196 | "sha256:0314a69e37426e3608aada02473b4161d4caf5a4b244d1d0c48072b8fee7bacc", 197 | "sha256:19da64c18d2d851112f09c287f8d3dbbdf725ab0e569077efb6cdcbd3497c107" 198 | ], 199 | "version": "==1.2.0" 200 | }, 201 | "h11": { 202 | "hashes": [ 203 | "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d", 204 | "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761" 205 | ], 206 | "markers": "python_version >= '3.7'", 207 | "version": "==0.14.0" 208 | }, 209 | "h2": { 210 | "hashes": [ 211 | "sha256:03a46bcf682256c95b5fd9e9a99c1323584c3eec6440d379b9903d709476bc6d", 212 | "sha256:a83aca08fbe7aacb79fec788c9c0bac936343560ed9ec18b82a13a12c28d2abb" 213 | ], 214 | "markers": "python_full_version >= '3.6.1'", 215 | "version": "==4.1.0" 216 | }, 217 | "hiredis": { 218 | "hashes": [ 219 | "sha256:071c5814b850574036506a8118034f97c3cbf2fe9947ff45a27b07a48da56240", 220 | "sha256:08415ea74c1c29b9d6a4ca3dd0e810dc1af343c1d1d442e15ba133b11ab5be6a", 221 | "sha256:126623b03c31cb6ac3e0d138feb6fcc36dd43dd34fc7da7b7a0c38b5d75bc896", 222 | "sha256:14824e457e4f5cda685c3345d125da13949bcf3bb1c88eb5d248c8d2c3dee08f", 223 | "sha256:15c2a551f3b8a26f7940d6ee10b837810201754b8d7e6f6b1391655370882c5a", 224 | "sha256:17e938d9d3ee92e1adbff361706f1c36cc60eeb3e3eeca7a3a353eae344f4c91", 225 | "sha256:1cadb0ac7ba3babfd804e425946bec9717b320564a1390f163a54af9365a720a", 226 | "sha256:1d274d5c511dfc03f83f997d3238eaa9b6ee3f982640979f509373cced891e98", 227 | "sha256:20f509e3a1a20d6e5f5794fc37ceb21f70f409101fcfe7a8bde783894d51b369", 228 | "sha256:227c5b4bcb60f89008c275d596e4a7b6625a6b3c827b8a66ae582eace7051f71", 229 | "sha256:232d0a70519865741ba56e1dfefd160a580ae78c30a1517bad47b3cf95a3bc7d", 230 | "sha256:2443659c76b226267e2a04dbbb21bc2a3f91aa53bdc0c22964632753ae43a247", 231 | "sha256:2d7e459fe7313925f395148d36d9b7f4f8dac65be06e45d7af356b187cef65fc", 232 | "sha256:2fb9300959a0048138791f3d68359d61a788574ec9556bddf1fec07f2dbc5320", 233 | "sha256:334f2738700b20faa04a0d813366fb16ed17287430a6b50584161d5ad31ca6d7", 234 | "sha256:33a94d264e6e12a79d9bb8af333b01dc286b9f39c99072ab5fef94ce1f018e17", 235 | "sha256:33bc4721632ef9708fa44e5df0066053fccc8e65410a2c48573192517a533b48", 236 | "sha256:33ee3ea5cad3a8cb339352cd230b411eb437a2e75d7736c4899acab32056ccdb", 237 | "sha256:3753df5f873d473f055e1f8837bfad0bd3b277c86f3c9bf058c58f14204cd901", 238 | "sha256:3759f4789ae1913b7df278dfc9e8749205b7a106f888cd2903d19461e24a7697", 239 | "sha256:3b7fe075e91b9d9cff40eba4fb6a8eff74964d3979a39be9a9ef58b1b4cb3604", 240 | "sha256:3bf4b5bae472630c229518e4a814b1b68f10a3d9b00aeaec45f1a330f03a0251", 241 | "sha256:3f006c28c885deb99b670a5a66f367a175ab8955b0374029bad7111f5357dcd4", 242 | "sha256:3f5446068197b35a11ccc697720c41879c8657e2e761aaa8311783aac84cef20", 243 | "sha256:3fa6811a618653164f918b891a0fa07052bd71a799defa5c44d167cac5557b26", 244 | "sha256:46525fbd84523cac75af5bf524bc74aaac848beaf31b142d2df8a787d9b4bbc4", 245 | "sha256:477c34c4489666dc73cb5e89dafe2617c3e13da1298917f73d55aac4696bd793", 246 | "sha256:4b3e974ad15eb32b1f537730dea70b93a4c3db7b026de3ad2b59da49c6f7454d", 247 | "sha256:4c3b8be557e08b234774925622e196f0ee36fe4eab66cd19df934d3efd8f3743", 248 | "sha256:4e3e3e31423f888d396b1fc1f936936e52af868ac1ec17dd15e3eeba9dd4de24", 249 | "sha256:4e43e2b5acaad09cf48c032f7e4926392bb3a3f01854416cf6d82ebff94d5467", 250 | "sha256:4ed68a3b1ccb4313d2a42546fd7e7439ad4745918a48b6c9bcaa61e1e3e42634", 251 | "sha256:4f674e309cd055ee7a48304ceb8cf43265d859faf4d7d01d270ce45e976ae9d3", 252 | "sha256:50171f985e17970f87d5a29e16603d1e5b03bdbf5c2691a37e6c912942a6b657", 253 | "sha256:51341e70b467004dcbec3a6ce8c478d2d6241e0f6b01e4c56764afd5022e1e9d", 254 | "sha256:5a4bcef114fc071d5f52c386c47f35aae0a5b43673197b9288a15b584da8fa3a", 255 | "sha256:5a5c8019ff94988d56eb49b15de76fe83f6b42536d76edeb6565dbf7fe14b973", 256 | "sha256:5cda592405bbd29d53942e0389dc3fa77b49c362640210d7e94a10c14a677d4d", 257 | "sha256:5e6674a017629284ef373b50496d9fb1a89b85a20a7fa100ecd109484ec748e5", 258 | "sha256:5e7bb4dd524f50b71c20ef5a12bd61da9b463f8894b18a06130942fe31509881", 259 | "sha256:60c4e3c258eafaab21b174b17270a0cc093718d61cdbde8c03f85ec4bf835343", 260 | "sha256:61995eb826009d99ed8590747bc0da683a5f4fbb4faa8788166bf3810845cd5c", 261 | "sha256:61a72e4a523cdfc521762137559c08dfa360a3caef63620be58c699d1717dac1", 262 | "sha256:69536b821dd1bc78058a6e7541743f8d82bf2d981b91280b14c4daa6cdc7faba", 263 | "sha256:6ccdcb635dae85b006592f78e32d97f4bc7541cb27829d505f9c7fefcef48298", 264 | "sha256:6f88cafe46612b6fa68e6dea49e25bebf160598bba00101caa51cc8c1f18d597", 265 | "sha256:6f969edc851efe23010e0f53a64269f2629a9364135e9ec81c842e8b2277d0c1", 266 | "sha256:77924b0d32fd1f493d3df15d9609ddf9d94c31a364022a6bf6b525ce9da75bea", 267 | "sha256:7df645b6b7800e8b748c217fbd6a4ca8361bcb9a1ae6206cc02377833ec8a1aa", 268 | "sha256:7e17d04ea58ab8cf3f2dc52e875db16077c6357846006780086fff3189fb199d", 269 | "sha256:7f2b34a6444b8f9c1e9f84bd2c639388e5d14f128afd14a869dfb3d9af893aa2", 270 | "sha256:818dfd310aa1020a13cd08ee48e116dd8c3bb2e23b8161f8ac4df587dd5093d7", 271 | "sha256:89a258424158eb8b3ed9f65548d68998da334ef155d09488c5637723eb1cd697", 272 | "sha256:8eceffca3941775b646cd585cd19b275d382de43cc3327d22f7c75d7b003d481", 273 | "sha256:8f280ab4e043b089777b43b4227bdc2035f88da5072ab36588e0ccf77d45d058", 274 | "sha256:8f9dbe12f011a9b784f58faecc171d22465bb532c310bd588d769ba79a59ef5a", 275 | "sha256:9076ce8429785c85f824650735791738de7143f61f43ae9ed83e163c0ca0fa44", 276 | "sha256:95d2305fd2a7b179cacb48b10f618872fc565c175f9f62b854e8d1acac3e8a9e", 277 | "sha256:96d9ea6c8d4cbdeee2e0d43379ce2881e4af0454b00570677c59f33f2531cd38", 278 | "sha256:9944a2cac25ffe049a7e89f306e11b900640837d1ef38d9be0eaa4a4e2b73a52", 279 | "sha256:9a1a80a8fa767f2fdc3870316a54b84fe9fc09fa6ab6a2686783de6a228a4604", 280 | "sha256:9cd32326dfa6ce87edf754153b0105aca64486bebe93b9600ccff74fa0b224df", 281 | "sha256:9f4a65276f6ecdebe75f2a53f578fbc40e8d2860658420d5e0611c56bbf5054c", 282 | "sha256:a286ded34eb16501002e3713b3130c987366eee2ba0d58c33c72f27778e31676", 283 | "sha256:a2df98f5e071320c7d84e8bd07c0542acdd0a7519307fc31774d60e4b842ec4f", 284 | "sha256:a7205497d7276a81fe92951a29616ef96562ed2f91a02066f72b6f93cb34b40e", 285 | "sha256:aa17a3b22b3726d54d7af20394f65d4a1735a842a4e0f557dc67a90f6965c4bc", 286 | "sha256:af33f370be90b48bbaf0dab32decbdcc522b1fa95d109020a963282086518a8e", 287 | "sha256:b17baf702c6e5b4bb66e1281a3efbb1d749c9d06cdb92b665ad81e03118f78fc", 288 | "sha256:b4f3d06dc16671b88a13ae85d8ca92534c0b637d59e49f0558d040a691246422", 289 | "sha256:b9953d87418ac228f508d93898ab572775e4d3b0eeb886a1a7734553bcdaf291", 290 | "sha256:b9a7c987e161e3c58f992c63b7e26fea7fe0777f3b975799d23d65bbb8cb5899", 291 | "sha256:c6cb613148422c523945cdb8b6bed617856f2602fd8750e33773ede2616e55d5", 292 | "sha256:c9b9e5bde7030cae83aa900b5bd660decc65afd2db8c400f3c568c815a47ca2a", 293 | "sha256:cc36a9dded458d4e37492fe3e619c6c83caae794d26ad925adbce61d592f8428", 294 | "sha256:cd2614f17e261f72efc2f19f5e5ff2ee19e2296570c0dcf33409e22be30710de", 295 | "sha256:d115790f18daa99b5c11a506e48923b630ef712e9e4b40482af942c3d40638b8", 296 | "sha256:d194decd9608f11c777946f596f31d5aacad13972a0a87829ae1e6f2d26c1885", 297 | "sha256:d1a4ce40ba11da9382c14da31f4f9e88c18f7d294f523decd0fadfb81f51ad18", 298 | "sha256:d1be9e30e675f5bc1cb534633324578f6f0944a1bcffe53242cf632f554f83b6", 299 | "sha256:d20891e3f33803b26d54c77fd5745878497091e33f4bbbdd454cf6e71aee8890", 300 | "sha256:d27e560eefb57914d742a837f1da98d3b29cb22eff013c8023b7cf52ae6e051d", 301 | "sha256:dcb0569dd5bfe6004658cd0f229efa699a3169dcb4f77bd72e188adda302063d", 302 | "sha256:e62ec131816c6120eff40dffe43424e140264a15fa4ab88c301bd6a595913af3", 303 | "sha256:e75163773a309e56a9b58165cf5a50e0f84b755f6ff863b2c01a38918fe92daa", 304 | "sha256:ec58fb7c2062f835595c12f0f02dcda76d0eb0831423cc191d1e18c9276648de", 305 | "sha256:f1eadbcd3de55ac42310ff82550d3302cb4efcd4e17d76646a17b6e7004bb42b", 306 | "sha256:f2dcb8389fa3d453927b1299f46bdb38473c293c8269d5c777d33ea0e526b610", 307 | "sha256:ffaf841546905d90ff189de7397aa56413b1ce5e54547f17a98f0ebf3a3b0a3b" 308 | ], 309 | "version": "==2.2.3" 310 | }, 311 | "hpack": { 312 | "hashes": [ 313 | "sha256:84a076fad3dc9a9f8063ccb8041ef100867b1878b25ef0ee63847a5d53818a6c", 314 | "sha256:fc41de0c63e687ebffde81187a948221294896f6bdc0ae2312708df339430095" 315 | ], 316 | "markers": "python_full_version >= '3.6.1'", 317 | "version": "==4.0.0" 318 | }, 319 | "httpcore": { 320 | "hashes": [ 321 | "sha256:125f8375ab60036db632f34f4b627a9ad085048eef7cb7d2616fea0f739f98af", 322 | "sha256:5581b9c12379c4288fe70f43c710d16060c10080617001e6b22a3b6dbcbefd36" 323 | ], 324 | "markers": "python_version >= '3.7'", 325 | "version": "==0.17.2" 326 | }, 327 | "httpx": { 328 | "hashes": [ 329 | "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd", 330 | "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd" 331 | ], 332 | "index": "pypi", 333 | "version": "==0.24.1" 334 | }, 335 | "hypercorn": { 336 | "hashes": [ 337 | "sha256:4a87a0b7bbe9dc75fab06dbe4b301b9b90416e9866c23a377df21a969d6ab8dd", 338 | "sha256:7c491d5184f28ee960dcdc14ab45d14633ca79d72ddd13cf4fcb4cb854d679ab" 339 | ], 340 | "index": "pypi", 341 | "version": "==0.14.3" 342 | }, 343 | "hyperframe": { 344 | "hashes": [ 345 | "sha256:0ec6bafd80d8ad2195c4f03aacba3a8265e57bc4cff261e802bf39970ed02a15", 346 | "sha256:ae510046231dc8e9ecb1a6586f63d2347bf4c8905914aa84ba585ae85f28a914" 347 | ], 348 | "markers": "python_full_version >= '3.6.1'", 349 | "version": "==6.0.1" 350 | }, 351 | "idna": { 352 | "hashes": [ 353 | "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", 354 | "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" 355 | ], 356 | "markers": "python_version >= '3.5'", 357 | "version": "==3.4" 358 | }, 359 | "importlib-metadata": { 360 | "hashes": [ 361 | "sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4", 362 | "sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5" 363 | ], 364 | "markers": "python_version < '3.10'", 365 | "version": "==6.7.0" 366 | }, 367 | "ipython": { 368 | "hashes": [ 369 | "sha256:1d197b907b6ba441b692c48cf2a3a2de280dc0ac91a3405b39349a50272ca0a1", 370 | "sha256:248aca623f5c99a6635bc3857677b7320b9b8039f99f070ee0d20a5ca5a8e6bf" 371 | ], 372 | "index": "pypi", 373 | "version": "==8.14.0" 374 | }, 375 | "itsdangerous": { 376 | "hashes": [ 377 | "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44", 378 | "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a" 379 | ], 380 | "markers": "python_version >= '3.7'", 381 | "version": "==2.1.2" 382 | }, 383 | "jedi": { 384 | "hashes": [ 385 | "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e", 386 | "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612" 387 | ], 388 | "markers": "python_version >= '3.6'", 389 | "version": "==0.18.2" 390 | }, 391 | "jinja2": { 392 | "hashes": [ 393 | "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", 394 | "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" 395 | ], 396 | "markers": "python_version >= '3.7'", 397 | "version": "==3.1.2" 398 | }, 399 | "langcodes": { 400 | "hashes": [ 401 | "sha256:4d89fc9acb6e9c8fdef70bcdf376113a3db09b67285d9e1d534de6d8818e7e69", 402 | "sha256:794d07d5a28781231ac335a1561b8442f8648ca07cd518310aeb45d6f0807ef6" 403 | ], 404 | "index": "pypi", 405 | "version": "==3.3.0" 406 | }, 407 | "language-data": { 408 | "hashes": [ 409 | "sha256:c1f5283c46bba68befa37505857a3f672497aba0c522b37d99367e911232455b", 410 | "sha256:f7ba86fafe099ef213ef597eda483d5227b12446604a61f617122d6c925847d5" 411 | ], 412 | "index": "pypi", 413 | "version": "==1.1" 414 | }, 415 | "marisa-trie": { 416 | "hashes": [ 417 | "sha256:0555104fe9f414abb12e967322a13df778b21958d1727470f4c8dedfde76a8f2", 418 | "sha256:07c14c88fde8a0ac55139f9fe763dc0deabc4b7950047719ae986ca62135e1fb", 419 | "sha256:08858920d0e09ca07d239252884fd72db2abb56c35ff463145ffc9c1277a4f34", 420 | "sha256:0ef2c4a5023bb6ddbaf1803187b7fb3108e9955aa9c60564504e5f622517c9e7", 421 | "sha256:1ae35c696f3c5b57c5fe4f73725102f3fe884bc658b854d484dfe6d7e72c86f5", 422 | "sha256:24e873619f61bef6a87c669ae459b79d98822270e8a10b21fc52dddf2acc9a46", 423 | "sha256:266bf4b6e00b4cff2b8618533919d38b883127f4e5c0af0e0bd78a042093dd99", 424 | "sha256:2f1cf9d5ead4471b149fdb93a1c84eddaa941d23e67b0782091adc222d198a87", 425 | "sha256:34189c321f30cefb76a6b20c7f055b3f6cd0bc8378c16ba8b7283fd898bf4ac2", 426 | "sha256:34f927f2738d0b402b76821895254e6a164d5020042559f7d910f6632829cdfa", 427 | "sha256:353113e811ccfa176fbb611b83671f0b3b40f46b3896b096c10e43f65d35916d", 428 | "sha256:396555d5f52dc86c65717052573fa2875e10f9e5dd014f825677beadcaec8248", 429 | "sha256:43abd082a21295b04859705b088d15acac8956587557680850e3149a79e36789", 430 | "sha256:45b0a38e015d0149141f028b8892ab518946b828c7931685199549294f5893ca", 431 | "sha256:49131e51aad530e4d47c716cef1bbef15a4e5b8f75bddfcdd7903f5043ef2331", 432 | "sha256:4ed76391b132c6261cfb402c1a08679e635d09a0a142dae2c1744d816f103c7f", 433 | "sha256:524c02f398d361aaf85d8f7709b5ac6de68d020c588fb6c087fb171137643c13", 434 | "sha256:55a5aea422a4c0c9ef143d3703323f2a43b4a5315fc90bbb6e9ff18544b8d931", 435 | "sha256:579d69981b18f427bd8e540199c4de400a2bd4ae98e96c814a12cbf766e7029b", 436 | "sha256:5c2a33ede2655f1a6fb840729128cb4bc48829108711f79b7a645b6c0c54b5c2", 437 | "sha256:5cf04156f38dc46f0f14423f98559c5def7d83f3a30f8a580c27ad3b0311ce76", 438 | "sha256:5f280f059be417cff81ac030db6a002f8a93093c7ca4555e570d43a24ed45514", 439 | "sha256:6412c816be723a0f11dd41225a30a08182cf2b3b7b3c882c44335003bde47003", 440 | "sha256:645908879ae8fcadfb51650fc176902b9e68eee9a8c4d4d8c682cf99ce3ff029", 441 | "sha256:66b13382be3c277f32143e6c814344118721c7954b2bfb57f5cfe93d17e63c9e", 442 | "sha256:68087942e95acb5801f2a5e9a874aa57af27a4afb52aca81fe1cbe22b2a2fd38", 443 | "sha256:6c1daaa8c38423fbd119db6654f92740d5ee40d1185a2bbc47afae6712b9ebfc", 444 | "sha256:6fcdb7f802db43857df3825c4c11acd14bb380deb961ff91e260950886531400", 445 | "sha256:71ed6286e9d593dac035b8516e7ec35a1b54a7d9c6451a9319e918a8ef722714", 446 | "sha256:7200cde8e2040811e98661a60463b296b76a6b224411f8899aa0850085e6af40", 447 | "sha256:73296b4d6d8ce2f6bc3898fe84348756beddb10cb56442391d050bff135e9c4c", 448 | "sha256:75317347f20bf05ab2ce5537a90989b1439b5e1752f558aad7b5d6b43194429b", 449 | "sha256:782c1515caa603656e15779bc61d5db3b079fa4270ad77f464908796e0d940aa", 450 | "sha256:80b22bdbebc3e6677e83db1352e4f6d478364107874c031a34a961437ead4e93", 451 | "sha256:82ba3caed5acfdff6a23d6881cc1927776b7320415261b6b24f48d0a190ab890", 452 | "sha256:84991b52a187d09b269c4caefc8b857a81156c44997eec7eac0e2862d108cc20", 453 | "sha256:891be5569cd6e3a059c2de53d63251aaaef513d68e8d2181f71378f9cb69e1ab", 454 | "sha256:8ccb3ba8a2a589b8a7aed693d564f20a6d3bbbb552975f904ba311cea6b85706", 455 | "sha256:9031184fe2215b591a6cdefe5d6d4901806fd7359e813c485a7ff25ea69d603c", 456 | "sha256:93172a7314d4d5993970dbafb746f23140d3abfa0d93cc174e766a302d125f7d", 457 | "sha256:a1b4d07158a3f9b4e84ee709a1fa86b9e11f3dd3b1e6fc45493195105a029545", 458 | "sha256:a432607bae139183c7251da7eb22f761440bc07d92eacc9e9f7dc0d87f70c495", 459 | "sha256:a537e0efff1ec880bc212390e97f1d35832a44bd78c96807ddb685d538875096", 460 | "sha256:a5bf2912810e135ce1e60a9b56a179ed62258306103bf5dd3186307f5c51b28f", 461 | "sha256:a891d2841da153b98c6c7fbe0a89ea8edbc164bdc96a001f360bdcdd54e2070d", 462 | "sha256:aee3de5f2836074cfd803f1caf16f68390f262ef09cd7dc7d0e8aee9b6878643", 463 | "sha256:bd86212d5037973deda057fc29d60e83dca05e68fa1e7ceaf014c513975c7a0d", 464 | "sha256:bfe649b02b6318bac572b86d9ddd8276c594411311f8e5ef2edc4bcd7285a06f", 465 | "sha256:c53b1d02f4974ecb52c6e8c6f4f1dbf3a15e79bc3861f4ad48b14e4e77c82342", 466 | "sha256:c8df5238c7b29498f4ee24fd3ee25e0129b3c56beaed1dd1628bce0ebac8ec8c", 467 | "sha256:c9ab632c5caef23a59cd43c76ab59e325f9eadd1e9c8b1c34005b9756ae716ee", 468 | "sha256:cc1c1dca06c0fdcca5bb261a09eca2b3bcf41eaeb467caf600ac68e77d3ed2c0", 469 | "sha256:d0d891f0138e5aecc9c5afb7b0a57c758e22c5b5c7c0edb0a1f21ae933259815", 470 | "sha256:d19f363b981fe9b4a302060a8088fd1f00906bc315db24f5d6726b5c309cc47e", 471 | "sha256:d37ea556bb99d9b0dfbe8fd6bdb17e91b91d04531be9e3b8b1b7b7f76ea55637", 472 | "sha256:d75b5d642b3d1e47a0ab649fb5eb6bf3681a5e1d3793c8ea7546586ab72731fd", 473 | "sha256:db2bdc480d83a1a566b3a64027f9fb34eae98bfe45788c41a45e99d430cbf48a", 474 | "sha256:e0d51c31fb41b6bc76c1abb7cf2d63a6e0ba7feffc96ea3d92b4d5084d71721a", 475 | "sha256:e6232506b4d66da932f70cf359a4c5ba9e086228ccd97b602159e90c6ea53dab", 476 | "sha256:f0359f392679774d1ff014f12efdf48da5d661e6241531ff55a3ae5a72a1137e", 477 | "sha256:f49a2cba047e643e5cd295d75de59f1df710c5e919cd376ac06ead513439881b", 478 | "sha256:f96531013252bca14f7665f67aa642be113b6c348ada5e167ebf8db27b1551b5", 479 | "sha256:fd7e71d8d85d04d2a5d23611663b2d322b60c98c2edab7e9ef9a2019f7435c5b" 480 | ], 481 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", 482 | "version": "==0.7.8" 483 | }, 484 | "markupsafe": { 485 | "hashes": [ 486 | "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e", 487 | "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e", 488 | "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431", 489 | "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686", 490 | "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559", 491 | "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc", 492 | "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c", 493 | "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0", 494 | "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4", 495 | "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9", 496 | "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575", 497 | "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba", 498 | "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d", 499 | "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3", 500 | "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00", 501 | "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155", 502 | "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac", 503 | "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52", 504 | "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f", 505 | "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8", 506 | "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b", 507 | "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24", 508 | "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea", 509 | "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198", 510 | "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0", 511 | "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee", 512 | "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be", 513 | "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2", 514 | "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707", 515 | "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6", 516 | "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58", 517 | "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779", 518 | "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636", 519 | "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c", 520 | "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad", 521 | "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee", 522 | "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc", 523 | "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2", 524 | "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48", 525 | "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7", 526 | "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e", 527 | "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b", 528 | "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa", 529 | "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5", 530 | "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e", 531 | "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb", 532 | "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9", 533 | "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57", 534 | "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc", 535 | "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2" 536 | ], 537 | "markers": "python_version >= '3.7'", 538 | "version": "==2.1.3" 539 | }, 540 | "matplotlib-inline": { 541 | "hashes": [ 542 | "sha256:f1f41aab5328aa5aaea9b16d083b128102f8712542f819fe7e6a420ff581b311", 543 | "sha256:f887e5f10ba98e8d2b150ddcf4702c1e5f8b3a20005eb0f74bfdbd360ee6f304" 544 | ], 545 | "markers": "python_version >= '3.5'", 546 | "version": "==0.1.6" 547 | }, 548 | "parso": { 549 | "hashes": [ 550 | "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0", 551 | "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75" 552 | ], 553 | "markers": "python_version >= '3.6'", 554 | "version": "==0.8.3" 555 | }, 556 | "pexpect": { 557 | "hashes": [ 558 | "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937", 559 | "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c" 560 | ], 561 | "markers": "sys_platform != 'win32'", 562 | "version": "==4.8.0" 563 | }, 564 | "pickleshare": { 565 | "hashes": [ 566 | "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca", 567 | "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56" 568 | ], 569 | "version": "==0.7.5" 570 | }, 571 | "priority": { 572 | "hashes": [ 573 | "sha256:6f8eefce5f3ad59baf2c080a664037bb4725cd0a790d53d59ab4059288faf6aa", 574 | "sha256:c965d54f1b8d0d0b19479db3924c7c36cf672dbf2aec92d43fbdaf4492ba18c0" 575 | ], 576 | "markers": "python_full_version >= '3.6.1'", 577 | "version": "==2.0.0" 578 | }, 579 | "prompt-toolkit": { 580 | "hashes": [ 581 | "sha256:23ac5d50538a9a38c8bde05fecb47d0b403ecd0662857a86f886f798563d5b9b", 582 | "sha256:45ea77a2f7c60418850331366c81cf6b5b9cf4c7fd34616f733c5427e6abbb1f" 583 | ], 584 | "markers": "python_full_version >= '3.7.0'", 585 | "version": "==3.0.38" 586 | }, 587 | "ptyprocess": { 588 | "hashes": [ 589 | "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", 590 | "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220" 591 | ], 592 | "version": "==0.7.0" 593 | }, 594 | "pure-eval": { 595 | "hashes": [ 596 | "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350", 597 | "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3" 598 | ], 599 | "version": "==0.2.2" 600 | }, 601 | "pygments": { 602 | "hashes": [ 603 | "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c", 604 | "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1" 605 | ], 606 | "markers": "python_version >= '3.7'", 607 | "version": "==2.15.1" 608 | }, 609 | "quart": { 610 | "hashes": [ 611 | "sha256:578a466bcd8c58b947b384ca3517c2a2f3bfeec8f58f4ff5038d4506ffee6be7", 612 | "sha256:c1766f269cdb85daf9da67ba54170abf7839aca97304dcb4cd0778eabfb442c6" 613 | ], 614 | "index": "pypi", 615 | "version": "==0.18.4" 616 | }, 617 | "quart-cors": { 618 | "hashes": [ 619 | "sha256:a12cb8f82506be9794c7d0fba62be04f07ca719e47e0691bf7a63d5ce661b70e", 620 | "sha256:e7c3f176624cfaa934ea96eddbcaea0b6c225d8eee543f97bce3bdc22e1e00ef" 621 | ], 622 | "index": "pypi", 623 | "version": "==0.6.0" 624 | }, 625 | "redis": { 626 | "hashes": [ 627 | "sha256:585dc516b9eb042a619ef0a39c3d7d55fe81bdb4df09a52c9cdde0d07bf1aa7d", 628 | "sha256:e2b03db868160ee4591de3cb90d40ebb50a90dd302138775937f6a42b7ed183c" 629 | ], 630 | "index": "pypi", 631 | "version": "==4.6.0" 632 | }, 633 | "regex": { 634 | "hashes": [ 635 | "sha256:0385e73da22363778ef2324950e08b689abdf0b108a7d8decb403ad7f5191938", 636 | "sha256:051da80e6eeb6e239e394ae60704d2b566aa6a7aed6f2890a7967307267a5dc6", 637 | "sha256:05ed27acdf4465c95826962528f9e8d41dbf9b1aa8531a387dee6ed215a3e9ef", 638 | "sha256:0654bca0cdf28a5956c83839162692725159f4cda8d63e0911a2c0dc76166525", 639 | "sha256:09e4a1a6acc39294a36b7338819b10baceb227f7f7dbbea0506d419b5a1dd8af", 640 | "sha256:0b49c764f88a79160fa64f9a7b425620e87c9f46095ef9c9920542ab2495c8bc", 641 | "sha256:0b71e63226e393b534105fcbdd8740410dc6b0854c2bfa39bbda6b0d40e59a54", 642 | "sha256:0c29ca1bd61b16b67be247be87390ef1d1ef702800f91fbd1991f5c4421ebae8", 643 | "sha256:10590510780b7541969287512d1b43f19f965c2ece6c9b1c00fc367b29d8dce7", 644 | "sha256:10cb847aeb1728412c666ab2e2000ba6f174f25b2bdc7292e7dd71b16db07568", 645 | "sha256:12b74fbbf6cbbf9dbce20eb9b5879469e97aeeaa874145517563cca4029db65c", 646 | "sha256:20326216cc2afe69b6e98528160b225d72f85ab080cbdf0b11528cbbaba2248f", 647 | "sha256:2239d95d8e243658b8dbb36b12bd10c33ad6e6933a54d36ff053713f129aa536", 648 | "sha256:25be746a8ec7bc7b082783216de8e9473803706723b3f6bef34b3d0ed03d57e2", 649 | "sha256:271f0bdba3c70b58e6f500b205d10a36fb4b58bd06ac61381b68de66442efddb", 650 | "sha256:29cdd471ebf9e0f2fb3cac165efedc3c58db841d83a518b082077e612d3ee5df", 651 | "sha256:2d44dc13229905ae96dd2ae2dd7cebf824ee92bc52e8cf03dcead37d926da019", 652 | "sha256:3676f1dd082be28b1266c93f618ee07741b704ab7b68501a173ce7d8d0d0ca18", 653 | "sha256:36efeba71c6539d23c4643be88295ce8c82c88bbd7c65e8a24081d2ca123da3f", 654 | "sha256:3e5219bf9e75993d73ab3d25985c857c77e614525fac9ae02b1bebd92f7cecac", 655 | "sha256:43e1dd9d12df9004246bacb79a0e5886b3b6071b32e41f83b0acbf293f820ee8", 656 | "sha256:457b6cce21bee41ac292d6753d5e94dcbc5c9e3e3a834da285b0bde7aa4a11e9", 657 | "sha256:463b6a3ceb5ca952e66550a4532cef94c9a0c80dc156c4cc343041951aec1697", 658 | "sha256:4959e8bcbfda5146477d21c3a8ad81b185cd252f3d0d6e4724a5ef11c012fb06", 659 | "sha256:4d3850beab9f527f06ccc94b446c864059c57651b3f911fddb8d9d3ec1d1b25d", 660 | "sha256:5708089ed5b40a7b2dc561e0c8baa9535b77771b64a8330b684823cfd5116036", 661 | "sha256:5c6b48d0fa50d8f4df3daf451be7f9689c2bde1a52b1225c5926e3f54b6a9ed1", 662 | "sha256:61474f0b41fe1a80e8dfa70f70ea1e047387b7cd01c85ec88fa44f5d7561d787", 663 | "sha256:6343c6928282c1f6a9db41f5fd551662310e8774c0e5ebccb767002fcf663ca9", 664 | "sha256:65ba8603753cec91c71de423a943ba506363b0e5c3fdb913ef8f9caa14b2c7e0", 665 | "sha256:687ea9d78a4b1cf82f8479cab23678aff723108df3edeac098e5b2498879f4a7", 666 | "sha256:6b2675068c8b56f6bfd5a2bda55b8accbb96c02fd563704732fd1c95e2083461", 667 | "sha256:7117d10690c38a622e54c432dfbbd3cbd92f09401d622902c32f6d377e2300ee", 668 | "sha256:7178bbc1b2ec40eaca599d13c092079bf529679bf0371c602edaa555e10b41c3", 669 | "sha256:72d1a25bf36d2050ceb35b517afe13864865268dfb45910e2e17a84be6cbfeb0", 670 | "sha256:742e19a90d9bb2f4a6cf2862b8b06dea5e09b96c9f2df1779e53432d7275331f", 671 | "sha256:74390d18c75054947e4194019077e243c06fbb62e541d8817a0fa822ea310c14", 672 | "sha256:74419d2b50ecb98360cfaa2974da8689cb3b45b9deff0dcf489c0d333bcc1477", 673 | "sha256:824bf3ac11001849aec3fa1d69abcb67aac3e150a933963fb12bda5151fe1bfd", 674 | "sha256:83320a09188e0e6c39088355d423aa9d056ad57a0b6c6381b300ec1a04ec3d16", 675 | "sha256:837328d14cde912af625d5f303ec29f7e28cdab588674897baafaf505341f2fc", 676 | "sha256:841d6e0e5663d4c7b4c8099c9997be748677d46cbf43f9f471150e560791f7ff", 677 | "sha256:87b2a5bb5e78ee0ad1de71c664d6eb536dc3947a46a69182a90f4410f5e3f7dd", 678 | "sha256:890e5a11c97cf0d0c550eb661b937a1e45431ffa79803b942a057c4fb12a2da2", 679 | "sha256:8abbc5d54ea0ee80e37fef009e3cec5dafd722ed3c829126253d3e22f3846f1e", 680 | "sha256:8e3f1316c2293e5469f8f09dc2d76efb6c3982d3da91ba95061a7e69489a14ef", 681 | "sha256:8f56fcb7ff7bf7404becdfc60b1e81a6d0561807051fd2f1860b0d0348156a07", 682 | "sha256:9427a399501818a7564f8c90eced1e9e20709ece36be701f394ada99890ea4b3", 683 | "sha256:976d7a304b59ede34ca2921305b57356694f9e6879db323fd90a80f865d355a3", 684 | "sha256:9a5bfb3004f2144a084a16ce19ca56b8ac46e6fd0651f54269fc9e230edb5e4a", 685 | "sha256:9beb322958aaca059f34975b0df135181f2e5d7a13b84d3e0e45434749cb20f7", 686 | "sha256:9edcbad1f8a407e450fbac88d89e04e0b99a08473f666a3f3de0fd292badb6aa", 687 | "sha256:9edce5281f965cf135e19840f4d93d55b3835122aa76ccacfd389e880ba4cf82", 688 | "sha256:a4c3b7fa4cdaa69268748665a1a6ff70c014d39bb69c50fda64b396c9116cf77", 689 | "sha256:a8105e9af3b029f243ab11ad47c19b566482c150c754e4c717900a798806b222", 690 | "sha256:a99b50300df5add73d307cf66abea093304a07eb017bce94f01e795090dea87c", 691 | "sha256:aad51907d74fc183033ad796dd4c2e080d1adcc4fd3c0fd4fd499f30c03011cd", 692 | "sha256:af4dd387354dc83a3bff67127a124c21116feb0d2ef536805c454721c5d7993d", 693 | "sha256:b28f5024a3a041009eb4c333863d7894d191215b39576535c6734cd88b0fcb68", 694 | "sha256:b4598b1897837067a57b08147a68ac026c1e73b31ef6e36deeeb1fa60b2933c9", 695 | "sha256:b6192d5af2ccd2a38877bfef086d35e6659566a335b1492786ff254c168b1693", 696 | "sha256:b862c2b9d5ae38a68b92e215b93f98d4c5e9454fa36aae4450f61dd33ff48487", 697 | "sha256:b956231ebdc45f5b7a2e1f90f66a12be9610ce775fe1b1d50414aac1e9206c06", 698 | "sha256:bb60b503ec8a6e4e3e03a681072fa3a5adcbfa5479fa2d898ae2b4a8e24c4591", 699 | "sha256:bbb02fd4462f37060122e5acacec78e49c0fbb303c30dd49c7f493cf21fc5b27", 700 | "sha256:bdff5eab10e59cf26bc479f565e25ed71a7d041d1ded04ccf9aee1d9f208487a", 701 | "sha256:c123f662be8ec5ab4ea72ea300359023a5d1df095b7ead76fedcd8babbedf969", 702 | "sha256:c2b867c17a7a7ae44c43ebbeb1b5ff406b3e8d5b3e14662683e5e66e6cc868d3", 703 | "sha256:c5f8037000eb21e4823aa485149f2299eb589f8d1fe4b448036d230c3f4e68e0", 704 | "sha256:c6a57b742133830eec44d9b2290daf5cbe0a2f1d6acee1b3c7b1c7b2f3606df7", 705 | "sha256:ccf91346b7bd20c790310c4147eee6ed495a54ddb6737162a36ce9dbef3e4751", 706 | "sha256:cf67ca618b4fd34aee78740bea954d7c69fdda419eb208c2c0c7060bb822d747", 707 | "sha256:d2da3abc88711bce7557412310dfa50327d5769a31d1c894b58eb256459dc289", 708 | "sha256:d4f03bb71d482f979bda92e1427f3ec9b220e62a7dd337af0aa6b47bf4498f72", 709 | "sha256:d54af539295392611e7efbe94e827311eb8b29668e2b3f4cadcfe6f46df9c777", 710 | "sha256:d77f09bc4b55d4bf7cc5eba785d87001d6757b7c9eec237fe2af57aba1a071d9", 711 | "sha256:d831c2f8ff278179705ca59f7e8524069c1a989e716a1874d6d1aab6119d91d1", 712 | "sha256:dbbbfce33cd98f97f6bffb17801b0576e653f4fdb1d399b2ea89638bc8d08ae1", 713 | "sha256:dcba6dae7de533c876255317c11f3abe4907ba7d9aa15d13e3d9710d4315ec0e", 714 | "sha256:e0bb18053dfcfed432cc3ac632b5e5e5c5b7e55fb3f8090e867bfd9b054dbcbf", 715 | "sha256:e2fbd6236aae3b7f9d514312cdb58e6494ee1c76a9948adde6eba33eb1c4264f", 716 | "sha256:e5087a3c59eef624a4591ef9eaa6e9a8d8a94c779dade95d27c0bc24650261cd", 717 | "sha256:e8915cc96abeb8983cea1df3c939e3c6e1ac778340c17732eb63bb96247b91d2", 718 | "sha256:ea353ecb6ab5f7e7d2f4372b1e779796ebd7b37352d290096978fea83c4dba0c", 719 | "sha256:ee2d1a9a253b1729bb2de27d41f696ae893507c7db224436abe83ee25356f5c1", 720 | "sha256:f415f802fbcafed5dcc694c13b1292f07fe0befdb94aa8a52905bd115ff41e88", 721 | "sha256:fb5ec16523dc573a4b277663a2b5a364e2099902d3944c9419a40ebd56a118f9", 722 | "sha256:fea75c3710d4f31389eed3c02f62d0b66a9da282521075061ce875eb5300cf23" 723 | ], 724 | "markers": "python_version >= '3.6'", 725 | "version": "==2023.6.3" 726 | }, 727 | "requests": { 728 | "hashes": [ 729 | "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", 730 | "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" 731 | ], 732 | "markers": "python_version >= '3.7'", 733 | "version": "==2.31.0" 734 | }, 735 | "setuptools": { 736 | "hashes": [ 737 | "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f", 738 | "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235" 739 | ], 740 | "markers": "python_version >= '3.7'", 741 | "version": "==68.0.0" 742 | }, 743 | "six": { 744 | "hashes": [ 745 | "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", 746 | "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" 747 | ], 748 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", 749 | "version": "==1.16.0" 750 | }, 751 | "sniffio": { 752 | "hashes": [ 753 | "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101", 754 | "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384" 755 | ], 756 | "markers": "python_version >= '3.7'", 757 | "version": "==1.3.0" 758 | }, 759 | "stack-data": { 760 | "hashes": [ 761 | "sha256:32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815", 762 | "sha256:cbb2a53eb64e5785878201a97ed7c7b94883f48b87bfb0bbe8b623c74679e4a8" 763 | ], 764 | "version": "==0.6.2" 765 | }, 766 | "strenum": { 767 | "hashes": [ 768 | "sha256:898cc0ebb5054ee07400341ac1d75fdfee489d76d6df3fbc1c2eaf95971e3916", 769 | "sha256:aebf04bba8e5af435937c452d69a86798b6f8d5ca5f20ba18561dbfad571ccdd" 770 | ], 771 | "index": "pypi", 772 | "version": "==0.4.10" 773 | }, 774 | "tenacity": { 775 | "hashes": [ 776 | "sha256:2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0", 777 | "sha256:43af037822bd0029025877f3b2d97cc4d7bb0c2991000a3d59d71517c5c969e0" 778 | ], 779 | "index": "pypi", 780 | "version": "==8.2.2" 781 | }, 782 | "tiktoken": { 783 | "hashes": [ 784 | "sha256:00d662de1e7986d129139faf15e6a6ee7665ee103440769b8dedf3e7ba6ac37f", 785 | "sha256:08efa59468dbe23ed038c28893e2a7158d8c211c3dd07f2bbc9a30e012512f1d", 786 | "sha256:176cad7f053d2cc82ce7e2a7c883ccc6971840a4b5276740d0b732a2b2011f8a", 787 | "sha256:1b6bce7c68aa765f666474c7c11a7aebda3816b58ecafb209afa59c799b0dd2d", 788 | "sha256:1e8fa13cf9889d2c928b9e258e9dbbbf88ab02016e4236aae76e3b4f82dd8288", 789 | "sha256:2ca30367ad750ee7d42fe80079d3092bd35bb266be7882b79c3bd159b39a17b0", 790 | "sha256:329f548a821a2f339adc9fbcfd9fc12602e4b3f8598df5593cfc09839e9ae5e4", 791 | "sha256:3dc3df19ddec79435bb2a94ee46f4b9560d0299c23520803d851008445671197", 792 | "sha256:450d504892b3ac80207700266ee87c932df8efea54e05cefe8613edc963c1285", 793 | "sha256:4d980fa066e962ef0f4dad0222e63a484c0c993c7a47c7dafda844ca5aded1f3", 794 | "sha256:55e251b1da3c293432179cf7c452cfa35562da286786be5a8b1ee3405c2b0dd2", 795 | "sha256:5727d852ead18b7927b8adf558a6f913a15c7766725b23dbe21d22e243041b28", 796 | "sha256:59b20a819969735b48161ced9b92f05dc4519c17be4015cfb73b65270a243620", 797 | "sha256:5a73286c35899ca51d8d764bc0b4d60838627ce193acb60cc88aea60bddec4fd", 798 | "sha256:64e1091c7103100d5e2c6ea706f0ec9cd6dc313e6fe7775ef777f40d8c20811e", 799 | "sha256:8d1d97f83697ff44466c6bef5d35b6bcdb51e0125829a9c0ed1e6e39fb9a08fb", 800 | "sha256:9c15d9955cc18d0d7ffcc9c03dc51167aedae98542238b54a2e659bd25fe77ed", 801 | "sha256:9c6dd439e878172dc163fced3bc7b19b9ab549c271b257599f55afc3a6a5edef", 802 | "sha256:9ec161e40ed44e4210d3b31e2ff426b4a55e8254f1023e5d2595cb60044f8ea6", 803 | "sha256:b1a038cee487931a5caaef0a2e8520e645508cde21717eacc9af3fbda097d8bb", 804 | "sha256:ba16698c42aad8190e746cd82f6a06769ac7edd415d62ba027ea1d99d958ed93", 805 | "sha256:bb2341836b725c60d0ab3c84970b9b5f68d4b733a7bcb80fb25967e5addb9920", 806 | "sha256:c06cd92b09eb0404cedce3702fa866bf0d00e399439dad3f10288ddc31045422", 807 | "sha256:c835d0ee1f84a5aa04921717754eadbc0f0a56cf613f78dfc1cf9ad35f6c3fea", 808 | "sha256:d0394967d2236a60fd0aacef26646b53636423cc9c70c32f7c5124ebe86f3093", 809 | "sha256:dae2af6f03ecba5f679449fa66ed96585b2fa6accb7fd57d9649e9e398a94f44", 810 | "sha256:e063b988b8ba8b66d6cc2026d937557437e79258095f52eaecfafb18a0a10c03", 811 | "sha256:e87751b54eb7bca580126353a9cf17a8a8eaadd44edaac0e01123e1513a33281", 812 | "sha256:f3020350685e009053829c1168703c346fb32c70c57d828ca3742558e94827a9" 813 | ], 814 | "index": "pypi", 815 | "version": "==0.4.0" 816 | }, 817 | "toml": { 818 | "hashes": [ 819 | "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", 820 | "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" 821 | ], 822 | "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", 823 | "version": "==0.10.2" 824 | }, 825 | "traitlets": { 826 | "hashes": [ 827 | "sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8", 828 | "sha256:f6cde21a9c68cf756af02035f72d5a723bf607e862e7be33ece505abf4a3bad9" 829 | ], 830 | "markers": "python_version >= '3.7'", 831 | "version": "==5.9.0" 832 | }, 833 | "typing-extensions": { 834 | "hashes": [ 835 | "sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26", 836 | "sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5" 837 | ], 838 | "markers": "python_version >= '3.7'", 839 | "version": "==4.6.3" 840 | }, 841 | "urllib3": { 842 | "hashes": [ 843 | "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1", 844 | "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825" 845 | ], 846 | "markers": "python_version >= '3.7'", 847 | "version": "==2.0.3" 848 | }, 849 | "wcwidth": { 850 | "hashes": [ 851 | "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e", 852 | "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0" 853 | ], 854 | "version": "==0.2.6" 855 | }, 856 | "werkzeug": { 857 | "hashes": [ 858 | "sha256:935539fa1413afbb9195b24880778422ed620c0fc09670945185cce4d91a8890", 859 | "sha256:98c774df2f91b05550078891dee5f0eb0cb797a522c757a2452b9cee5b202330" 860 | ], 861 | "markers": "python_version >= '3.8'", 862 | "version": "==2.3.6" 863 | }, 864 | "wsproto": { 865 | "hashes": [ 866 | "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065", 867 | "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736" 868 | ], 869 | "markers": "python_full_version >= '3.7.0'", 870 | "version": "==1.2.0" 871 | }, 872 | "youtube-transcript-api": { 873 | "hashes": [ 874 | "sha256:bc148ea687af7d8e80853d4cd005c7d28859106a5eb5501722fd3dac9f9b68be", 875 | "sha256:f0ce9c8475953c108d3cbfda0426dc35f195638bf841837968530d300c6c7ec5" 876 | ], 877 | "index": "pypi", 878 | "version": "==0.6.1" 879 | }, 880 | "zipp": { 881 | "hashes": [ 882 | "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b", 883 | "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556" 884 | ], 885 | "markers": "python_version >= '3.7'", 886 | "version": "==3.15.0" 887 | } 888 | }, 889 | "develop": { 890 | "autopep8": { 891 | "hashes": [ 892 | "sha256:86e9303b5e5c8160872b2f5ef611161b2893e9bfe8ccc7e2f76385947d57a2f1", 893 | "sha256:f9849cdd62108cb739dbcdbfb7fdcc9a30d1b63c4cc3e1c1f893b5360941b61c" 894 | ], 895 | "index": "pypi", 896 | "version": "==2.0.2" 897 | }, 898 | "pycodestyle": { 899 | "hashes": [ 900 | "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053", 901 | "sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610" 902 | ], 903 | "markers": "python_version >= '3.6'", 904 | "version": "==2.10.0" 905 | }, 906 | "tomli": { 907 | "hashes": [ 908 | "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", 909 | "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" 910 | ], 911 | "markers": "python_version < '3.11'", 912 | "version": "==2.0.1" 913 | } 914 | } 915 | } 916 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # better-youtube-summary-server 2 | 3 | Literally Better YouTube Summary 🎯 4 | 5 | [![Better YouTube Summary Extension Showcase](https://res.cloudinary.com/marcomontalbano/image/upload/v1707146334/video_to_markdown/images/youtube--NyhrKImPSDQ-c05b58ac6eb4c4700831b2b3070cd403.jpg)](https://www.youtube.com/watch?v=NyhrKImPSDQ "Better YouTube Summary Extension Showcase") 6 | 7 | **This project is no longer maintained,** 8 | 9 | because OpenAI banned my account due to "accessing the API from an [unsupported location](https://platform.openai.com/docs/supported-countries)" 👎 10 | 11 | The frontend implementation can be found in [mthli/better-youtube-summary-extension](https://github.com/mthli/better-youtube-summary-extension). 12 | 13 | If you want to deploy it yourself, please replace the `bys.mthli.com` with your own domain. 14 | 15 | ## Development 16 | 17 | Currently this project is developed on **macOS 13.3 (22E252).** 18 | 19 | But this project **can't run on macOS** actually, just for coding. 20 | 21 | First install dependencies as follow: 22 | 23 | ```bash 24 | # Install 'redis' if you don't have. 25 | # https://redis.io/docs/getting-started/installation/install-redis-on-mac-os 26 | brew install redis 27 | brew services start redis 28 | 29 | # Install 'python3' if you don't have. 30 | brew install python3 31 | 32 | # Install 'pyenv' if you don't have. 33 | # https://github.com/pyenv/pyenv#automatic-installer 34 | curl https://pyenv.run | bash 35 | 36 | # Install 'pipenv' if you don't have. 37 | pip3 install --user pipenv 38 | 39 | # Install all dependencies needed by this project. 40 | pipenv install 41 | pipenv install --dev 42 | ``` 43 | 44 | Then just open you editor and have fun. 45 | 46 | ## Deployment 47 | 48 | This project should be deployed to **Debian GNU/Linux 11 (bullseye).** 49 | 50 | First install dependencies as follow: 51 | 52 | ```bash 53 | # Install 'nginx' if you don't have. 54 | sudo apt-get install nginx 55 | sudo systemd enable nginx 56 | sudo systemd start nginx 57 | 58 | # Install 'redis' if you don't have. 59 | sudo apt-get install redis 60 | sudo systemd enable redis 61 | sudo systemd start redis 62 | 63 | # Install 'certbot' if you don't have. 64 | sudo apt-get install certbot 65 | sudo apt-get install python3-certbot-nginx 66 | 67 | # Install 'pm2' if you don't have. 68 | curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash 69 | nvm install node # restart your bash, then 70 | npm install -g pm2 71 | pm2 install pm2-logrotate 72 | 73 | # Install 'python3' if you don't have. 74 | sudo apt-get install python3 75 | sudo apt-get install python3-pip 76 | 77 | # Install 'pyenv' if you don't have. 78 | # https://github.com/pyenv/pyenv#automatic-installer 79 | curl https://pyenv.run | bash 80 | 81 | # Install 'pipenv' if you don't have. 82 | pip install --user pipenv 83 | 84 | # Install all dependencies needed by this project. 85 | pipenv install 86 | pipenv install --dev 87 | ``` 88 | 89 | Before run this project: 90 | 91 | - Set `openai_api_key` defined in `./rds.py` with `redis-cli` 92 | - Put `./bys.mthli.com.conf` to `/etc/nginx/conf.d/` directory 93 | - Execute `sudo certbot --nginx -d bys.mthli.com` to generate certificates, or 94 | - Execute `sudo certbot renew` to avoid certificates expired after 90 days 95 | 96 | Then just execute commands as follow: 97 | 98 | ```bash 99 | # Make sure you are not in pipenv shell. 100 | pm2 start ./pm2.json 101 | ``` 102 | 103 | ## License 104 | 105 | ``` 106 | better-youtube-summary-server - Literally Better YouTube Summary. 107 | 108 | Copyright (C) 2023 Matthew Lee 109 | 110 | This program is free software: you can redistribute it and/or modify 111 | it under the terms of the GNU Affero General Public License as published 112 | by the Free Software Foundation, either version 3 of the License, or 113 | (at your option) any later version. 114 | 115 | This program is distributed in the hope that it will be useful, 116 | but WITHOUT ANY WARRANTY; without even the implied warranty of 117 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 118 | GNU Affero General Public License for more details. 119 | 120 | You should have received a copy of the GNU Affero General Public License 121 | along with this program. If not, see . 122 | ``` 123 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | from dataclasses import asdict 2 | from uuid import uuid4 3 | 4 | from arq import create_pool 5 | from arq.connections import RedisSettings 6 | from arq.typing import WorkerSettingsBase 7 | from langcodes import Language 8 | from quart import Quart, Response, abort, json, request, make_response 9 | from quart_cors import cors 10 | from werkzeug.datastructures import Headers 11 | from werkzeug.exceptions import HTTPException 12 | from youtube_transcript_api import NoTranscriptFound, TranscriptsDisabled 13 | 14 | from constants import APPLICATION_JSON 15 | from database.chapter import \ 16 | create_chapter_table, \ 17 | find_chapters_by_vid, \ 18 | insert_chapters, \ 19 | delete_chapters_by_vid 20 | from database.data import \ 21 | ChapterSlicer, \ 22 | Feedback, \ 23 | State, \ 24 | TimedText, \ 25 | User 26 | from database.feedback import \ 27 | create_feedback_table, \ 28 | find_feedback, \ 29 | insert_or_update_feedback, \ 30 | delete_feedback 31 | from database.translation import create_translation_table, delete_translation 32 | from database.user import create_user_table, find_user, insert_or_update_user 33 | from logger import logger 34 | from rds import rds 35 | from sse import sse_subscribe 36 | from summary import \ 37 | SUMMARIZING_RDS_KEY_EX, \ 38 | NO_TRANSCRIPT_RDS_KEY_EX, \ 39 | build_summary_channel, \ 40 | build_summary_response, \ 41 | build_summarizing_rds_key, \ 42 | build_no_transcript_rds_key, \ 43 | do_if_found_chapters_in_database, \ 44 | need_to_resummarize, \ 45 | parse_timed_texts_and_lang, \ 46 | summarize as summarizing 47 | from translation import translate as translating 48 | 49 | app = Quart(__name__) 50 | app = cors(app, allow_origin='*') 51 | 52 | create_chapter_table() 53 | create_feedback_table() 54 | create_translation_table() 55 | create_user_table() 56 | 57 | 58 | # https://pgjones.gitlab.io/quart/how_to_guides/startup_shutdown.html 59 | @app.before_serving 60 | async def before_serving(): 61 | logger.info(f'create arq in app before serving') 62 | app.arq = await create_pool(RedisSettings()) 63 | 64 | 65 | # https://flask.palletsprojects.com/en/2.2.x/errorhandling/#generic-exception-handler 66 | # 67 | # If no handler is registered, 68 | # HTTPException subclasses show a generic message about their code, 69 | # while other exceptions are converted to a generic "500 Internal Server Error". 70 | @app.errorhandler(HTTPException) 71 | def handle_exception(e: HTTPException): 72 | response = e.get_response() 73 | response.data = json.dumps({ 74 | 'code': e.code, 75 | 'name': e.name, 76 | 'description': e.description, 77 | }) 78 | response.content_type = APPLICATION_JSON 79 | logger.error(f'errorhandler, data={response.data}') 80 | return response 81 | 82 | 83 | @app.post('/api/user') 84 | async def add_user(): 85 | uid = str(uuid4()) 86 | insert_or_update_user(User(uid=uid)) 87 | return { 88 | 'uid': uid, 89 | } 90 | 91 | 92 | # { 93 | # 'vid': str, required. 94 | # 'bad': bool, optional. 95 | # 'good': bool, optional. 96 | # } 97 | @app.post('/api/feedback/') 98 | async def feedback(vid: str): 99 | try: 100 | body: dict = await request.get_json() or {} 101 | except Exception as e: 102 | abort(400, f'feedback failed, e={e}') 103 | 104 | _ = _parse_uid_from_headers(request.headers) 105 | 106 | found = find_chapters_by_vid(vid=vid, limit=1) 107 | if not found: 108 | return {} 109 | 110 | feedback = find_feedback(vid) 111 | if not feedback: 112 | feedback = Feedback(vid=vid) 113 | 114 | good = body.get('good', False) 115 | if not isinstance(good, bool): 116 | abort(400, '"good" must be bool') 117 | if good: 118 | feedback.good += 1 119 | 120 | bad = body.get('bad', False) 121 | if not isinstance(bad, bool): 122 | abort(400, '"bad" must be bool') 123 | if bad: 124 | feedback.bad += 1 125 | 126 | insert_or_update_feedback(feedback) 127 | return {} 128 | 129 | 130 | # { 131 | # 'chapters': dict, optional. 132 | # 'no_transcript': boolean, optional. 133 | # } 134 | @app.post('/api/summarize/') 135 | async def summarize(vid: str): 136 | try: 137 | body: dict = await request.get_json() or {} 138 | except Exception as e: 139 | abort(400, f'summarize failed, e={e}') 140 | 141 | uid = _parse_uid_from_headers(request.headers) 142 | openai_api_key = _parse_openai_api_key_from_headers(request.headers) 143 | chapters = _parse_chapters_from_body(body) 144 | no_transcript = bool(body.get('no_transcript', False)) 145 | 146 | no_transcript_rds_key = build_no_transcript_rds_key(vid) 147 | summarizing_rds_key = build_summarizing_rds_key(vid) 148 | channel = build_summary_channel(vid) 149 | 150 | found = find_chapters_by_vid(vid) 151 | if found: 152 | if (chapters and found[0].slicer != ChapterSlicer.YOUTUBE) or \ 153 | need_to_resummarize(vid, found): 154 | logger.info(f'summarize, need to resummarize, vid={vid}') 155 | delete_chapters_by_vid(vid) 156 | delete_feedback(vid) 157 | delete_translation(vid) 158 | rds.delete(no_transcript_rds_key) 159 | rds.delete(summarizing_rds_key) 160 | else: 161 | logger.info(f'summarize, found chapters in database, vid={vid}') 162 | await do_if_found_chapters_in_database(vid, found) 163 | return build_summary_response(State.DONE, found) 164 | 165 | if rds.exists(no_transcript_rds_key) or no_transcript: 166 | logger.info(f'summarize, but no transcript for now, vid={vid}') 167 | return build_summary_response(State.NOTHING) 168 | 169 | if rds.exists(summarizing_rds_key): 170 | logger.info(f'summarize, but repeated, vid={vid}') 171 | return await _build_sse_response(channel) 172 | 173 | # Set the summary proccess beginning flag here, 174 | # because of we need to get the transcript first, 175 | # and try to avoid youtube rate limits. 176 | rds.set(summarizing_rds_key, 1, ex=SUMMARIZING_RDS_KEY_EX) 177 | 178 | try: 179 | # FIXME (Matthew Lee) youtube rate limits? 180 | timed_texts, lang = parse_timed_texts_and_lang(vid) 181 | if not timed_texts: 182 | logger.warning(f'summarize, but no transcript found, vid={vid}') 183 | rds.set(no_transcript_rds_key, 1, ex=NO_TRANSCRIPT_RDS_KEY_EX) 184 | rds.delete(summarizing_rds_key) 185 | return build_summary_response(State.NOTHING) 186 | except (NoTranscriptFound, TranscriptsDisabled): 187 | logger.warning(f'summarize, but no transcript found, vid={vid}') 188 | rds.set(no_transcript_rds_key, 1, ex=NO_TRANSCRIPT_RDS_KEY_EX) 189 | rds.delete(summarizing_rds_key) 190 | return build_summary_response(State.NOTHING) 191 | except Exception: 192 | logger.exception(f'summarize failed, vid={vid}') 193 | rds.delete(no_transcript_rds_key) 194 | rds.delete(summarizing_rds_key) 195 | raise # to errorhandler. 196 | 197 | await app.arq.enqueue_job( 198 | do_summarize_job.__name__, 199 | vid, 200 | uid, 201 | chapters, 202 | timed_texts, 203 | lang, 204 | openai_api_key, 205 | ) 206 | 207 | return await _build_sse_response(channel) 208 | 209 | 210 | # { 211 | # 'cid': str, required. 212 | # 'lang': str, required. 213 | # } 214 | @app.post('/api/translate/') 215 | async def translate(vid: str): 216 | _ = _parse_uid_from_headers(request.headers) 217 | openai_api_key = _parse_openai_api_key_from_headers(request.headers) 218 | 219 | try: 220 | body: dict = await request.get_json() or {} 221 | except Exception as e: 222 | abort(400, f'translate failed, e={e}') 223 | 224 | cid = body.get('cid', '') 225 | if not isinstance(cid, str): 226 | abort(400, f'"cid" must be string') 227 | cid = cid.strip() 228 | if not cid: 229 | abort(400, f'"cid" must not empty') 230 | 231 | lang = body.get('lang', '') 232 | if not isinstance(lang, str): 233 | abort(400, f'"lang" must be string') 234 | lang = lang.strip() 235 | if not lang: 236 | abort(400, f'"lang" must not empty') 237 | lang = Language.get(lang) # LanguageTagError. 238 | if not lang.is_valid(): 239 | abort(400, f'"lang" invalid') 240 | lang = lang.language # to str. 241 | 242 | trans = await translating( 243 | vid=vid, 244 | cid=cid, 245 | lang=lang, 246 | openai_api_key=openai_api_key, 247 | ) 248 | 249 | return asdict(trans) if trans else {} 250 | 251 | 252 | def _parse_uid_from_headers(headers: Headers, check: bool = True) -> str: 253 | uid = headers.get(key='uid', default='', type=str) 254 | if not isinstance(uid, str): 255 | abort(400, f'"uid" must be string') 256 | 257 | uid = uid.strip() 258 | if not uid: 259 | abort(400, f'"uid" must not empty') 260 | 261 | if check: 262 | user = find_user(uid=uid) 263 | if not user: 264 | abort(404, f'user not exists') 265 | if user.is_deleted: 266 | abort(404, f'user is deleted') 267 | 268 | return uid 269 | 270 | 271 | def _parse_openai_api_key_from_headers(headers: Headers) -> str: 272 | # Don't use underscore here because of Ngnix. 273 | openai_api_key = headers.get(key='openai-api-key', default='', type=str) 274 | if not isinstance(openai_api_key, str): 275 | abort(400, f'"openai-api-key" must be string') 276 | return openai_api_key.strip() 277 | 278 | 279 | def _parse_chapters_from_body(body: dict) -> list[dict]: 280 | chapters = body.get('chapters', []) 281 | if not isinstance(chapters, list): 282 | abort(400, f'"chapters" must be list') 283 | for c in chapters: 284 | if not isinstance(c, dict): 285 | abort(400, f'"chapters" item must be dict') 286 | return chapters 287 | 288 | 289 | # ctx is arq first param, keep it. 290 | async def do_on_arq_worker_startup(ctx: dict): 291 | logger.info(f'arq worker startup') 292 | 293 | 294 | # ctx is arq first param, keep it. 295 | async def do_on_arq_worker_shutdown(ctx: dict): 296 | logger.info(f'arq worker shutdown') 297 | 298 | 299 | # ctx is arq first param, keep it. 300 | async def do_summarize_job( 301 | ctx: dict, 302 | vid: str, 303 | trigger: str, 304 | chapters: list[dict], 305 | timed_texts: list[TimedText], 306 | lang: str, 307 | openai_api_key: str = '', 308 | ): 309 | logger.info(f'do summarize job, vid={vid}') 310 | 311 | # Set flag again, although we have done this before. 312 | summarizing_rds_key = build_summarizing_rds_key(vid) 313 | rds.set(summarizing_rds_key, 1, ex=SUMMARIZING_RDS_KEY_EX) 314 | 315 | chapters, _ = await summarizing( 316 | vid=vid, 317 | trigger=trigger, 318 | chapters=chapters, 319 | timed_texts=timed_texts, 320 | lang=lang, 321 | openai_api_key=openai_api_key, 322 | ) 323 | 324 | if chapters: 325 | logger.info(f'summarize, save chapters to database, vid={vid}') 326 | delete_chapters_by_vid(vid) 327 | delete_feedback(vid) 328 | delete_translation(vid) 329 | insert_chapters(chapters) 330 | 331 | rds.delete(build_no_transcript_rds_key(vid)) 332 | rds.delete(summarizing_rds_key) 333 | 334 | 335 | # https://quart.palletsprojects.com/en/latest/how_to_guides/server_sent_events.html 336 | async def _build_sse_response(channel: str) -> Response: 337 | res = await make_response( 338 | sse_subscribe(channel), 339 | { 340 | 'Content-Type': 'text/event-stream', 341 | 'Transfer-Encoding': 'chunked', 342 | 'Cache-Control': 'no-cache', 343 | 'X-Accel-Buffering': 'no', 344 | }, 345 | ) 346 | 347 | res.timeout = None 348 | return res 349 | 350 | 351 | # https://arq-docs.helpmanual.io/#simple-usage 352 | class WorkerSettings(WorkerSettingsBase): 353 | functions = [do_summarize_job] 354 | on_startup = do_on_arq_worker_startup 355 | on_shutdown = do_on_arq_worker_shutdown 356 | -------------------------------------------------------------------------------- /bys.mthli.com.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | listen [::]:80; 4 | server_name bys.mthli.com; 5 | 6 | location / { 7 | proxy_pass http://127.0.0.1:8000; 8 | 9 | proxy_http_version 1.1; 10 | proxy_redirect off; 11 | 12 | proxy_connect_timeout 300; 13 | proxy_read_timeout 300; 14 | proxy_send_timeout 300; 15 | 16 | proxy_set_header Host $host; 17 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 18 | proxy_set_header X-Forwarded-Host $host; 19 | proxy_set_header X-Forwarded-Prefix /; 20 | proxy_set_header X-Forwarded-Proto $scheme; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /constants.py: -------------------------------------------------------------------------------- 1 | APPLICATION_JSON = 'application/json' 2 | 3 | # https://www.whatismybrowser.com/guides/the-latest-user-agent/chrome 4 | USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36' 5 | -------------------------------------------------------------------------------- /database/chapter.py: -------------------------------------------------------------------------------- 1 | from sys import maxsize 2 | from typing import Optional 3 | 4 | from database.data import Chapter 5 | from database.sqlite import commit, fetchall, sqlescape 6 | 7 | _TABLE = 'chapter' 8 | _COLUMN_CID = 'cid' # UUID. 9 | _COLUMN_VID = 'vid' 10 | _COLUMN_TRIGGER = 'trigger' # uid. 11 | _COLUMN_SLICER = 'slicer' 12 | _COLUMN_STYLE = 'style' 13 | _COLUMN_START = 'start' # in seconds. 14 | _COLUMN_LANG = 'lang' # language code. 15 | _COLUMN_CHAPTER = 'chapter' 16 | _COLUMN_SUMMARY = 'summary' 17 | _COLUMN_REFINED = 'refined' 18 | _COLUMN_CREATE_TIMESTAMP = 'create_timestamp' 19 | _COLUMN_UPDATE_TIMESTAMP = 'update_timestamp' 20 | 21 | 22 | def create_chapter_table(): 23 | commit(f''' 24 | CREATE TABLE IF NOT EXISTS {_TABLE} ( 25 | {_COLUMN_CID} TEXT NOT NULL PRIMARY KEY, 26 | {_COLUMN_VID} TEXT NOT NULL DEFAULT '', 27 | {_COLUMN_TRIGGER} TEXT NOT NULL DEFAULT '', 28 | {_COLUMN_SLICER} TEXT NOT NULL DEFAULT '', 29 | {_COLUMN_STYLE} TEXT NOT NULL DEFAULT '', 30 | {_COLUMN_START} INTEGER NOT NULL DEFAULT 0, 31 | {_COLUMN_LANG} TEXT NOT NULL DEFAULT '', 32 | {_COLUMN_CHAPTER} TEXT NOT NULL DEFAULT '', 33 | {_COLUMN_SUMMARY} TEXT NOT NULL DEFAULT '', 34 | {_COLUMN_REFINED} INTEGER NOT NULL DEFAULT 0, 35 | {_COLUMN_CREATE_TIMESTAMP} INTEGER NOT NULL DEFAULT 0, 36 | {_COLUMN_UPDATE_TIMESTAMP} INTEGER NOT NULL DEFAULT 0 37 | ) 38 | ''') 39 | commit(f''' 40 | CREATE INDEX IF NOT EXISTS idx_{_COLUMN_TRIGGER} 41 | ON {_TABLE} ({_COLUMN_TRIGGER}) 42 | ''') 43 | commit(f''' 44 | CREATE INDEX IF NOT EXISTS idx_{_COLUMN_VID} 45 | ON {_TABLE} ({_COLUMN_VID}) 46 | ''') 47 | commit(f''' 48 | CREATE INDEX IF NOT EXISTS idx_{_COLUMN_CREATE_TIMESTAMP} 49 | ON {_TABLE} ({_COLUMN_CREATE_TIMESTAMP}) 50 | ''') 51 | commit(f''' 52 | CREATE INDEX IF NOT EXISTS idx_{_COLUMN_UPDATE_TIMESTAMP} 53 | ON {_TABLE} ({_COLUMN_UPDATE_TIMESTAMP}) 54 | ''') 55 | 56 | 57 | def find_chapter_by_cid(cid: str) -> Optional[Chapter]: 58 | res = fetchall(f''' 59 | SELECT 60 | {_COLUMN_CID}, 61 | {_COLUMN_VID}, 62 | {_COLUMN_TRIGGER}, 63 | {_COLUMN_SLICER}, 64 | {_COLUMN_STYLE}, 65 | {_COLUMN_START}, 66 | {_COLUMN_LANG}, 67 | {_COLUMN_CHAPTER}, 68 | {_COLUMN_SUMMARY}, 69 | {_COLUMN_REFINED} 70 | FROM {_TABLE} 71 | WHERE {_COLUMN_CID} = '{sqlescape(cid)}' 72 | LIMIT 1 73 | ''') 74 | 75 | if not res: 76 | return None 77 | 78 | res = res[0] 79 | return Chapter( 80 | cid=res[0], 81 | vid=res[1], 82 | trigger=res[2], 83 | slicer=res[3], 84 | style=res[4], 85 | start=res[5], 86 | lang=res[6], 87 | chapter=res[7], 88 | summary=res[8], 89 | refined=res[9], 90 | ) 91 | 92 | 93 | def find_chapters_by_vid(vid: str, limit: int = maxsize) -> list[Chapter]: 94 | res = fetchall(f''' 95 | SELECT 96 | {_COLUMN_CID}, 97 | {_COLUMN_VID}, 98 | {_COLUMN_TRIGGER}, 99 | {_COLUMN_SLICER}, 100 | {_COLUMN_STYLE}, 101 | {_COLUMN_START}, 102 | {_COLUMN_LANG}, 103 | {_COLUMN_CHAPTER}, 104 | {_COLUMN_SUMMARY}, 105 | {_COLUMN_REFINED} 106 | FROM {_TABLE} 107 | WHERE {_COLUMN_VID} = '{sqlescape(vid)}' 108 | ORDER BY {_COLUMN_START} ASC 109 | LIMIT {limit} 110 | ''') 111 | return list(map(lambda r: Chapter( 112 | cid=r[0], 113 | vid=r[1], 114 | trigger=r[2], 115 | slicer=r[3], 116 | style=r[4], 117 | start=r[5], 118 | lang=r[6], 119 | chapter=r[7], 120 | summary=r[8], 121 | refined=r[9], 122 | ), res)) 123 | 124 | 125 | def insert_chapters(chapters: list[Chapter]): 126 | for c in chapters: 127 | _insert_chapter(c) 128 | 129 | 130 | def _insert_chapter(chapter: Chapter): 131 | commit(f''' 132 | INSERT INTO {_TABLE} ( 133 | {_COLUMN_CID}, 134 | {_COLUMN_VID}, 135 | {_COLUMN_TRIGGER}, 136 | {_COLUMN_SLICER}, 137 | {_COLUMN_STYLE}, 138 | {_COLUMN_START}, 139 | {_COLUMN_LANG}, 140 | {_COLUMN_CHAPTER}, 141 | {_COLUMN_SUMMARY}, 142 | {_COLUMN_REFINED}, 143 | {_COLUMN_CREATE_TIMESTAMP}, 144 | {_COLUMN_UPDATE_TIMESTAMP} 145 | ) VALUES ( 146 | '{sqlescape(chapter.cid)}', 147 | '{sqlescape(chapter.vid)}', 148 | '{sqlescape(chapter.trigger)}', 149 | '{sqlescape(chapter.slicer)}', 150 | '{sqlescape(chapter.style)}', 151 | {chapter.start}, 152 | '{sqlescape(chapter.lang)}', 153 | '{sqlescape(chapter.chapter)}', 154 | '{sqlescape(chapter.summary)}', 155 | {chapter.refined}, 156 | STRFTIME('%s', 'NOW'), 157 | STRFTIME('%s', 'NOW') 158 | ) 159 | ''') 160 | 161 | 162 | def delete_chapters_by_vid(vid: str): 163 | commit(f''' 164 | DELETE FROM {_TABLE} 165 | WHERE {_COLUMN_VID} = '{sqlescape(vid)}' 166 | ''') 167 | -------------------------------------------------------------------------------- /database/data.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import unique 3 | 4 | from strenum import StrEnum 5 | 6 | 7 | @dataclass 8 | class Chapter: 9 | cid: str = '' # required. 10 | vid: str = '' # required. 11 | trigger: str = '' # required; uid. 12 | slicer: str = '' # required. 13 | style: str = '' # required. 14 | start: int = 0 # required; in seconds. 15 | lang: str = '' # required; language code. 16 | chapter: str = '' # required. 17 | summary: str = '' # optional. 18 | refined: int = 0 # optional. 19 | 20 | 21 | @unique 22 | class ChapterSlicer(StrEnum): 23 | YOUTUBE = 'youtube' 24 | OPENAI = 'openai' 25 | 26 | 27 | @unique 28 | class ChapterStyle(StrEnum): 29 | MARKDOWN = 'markdown' 30 | TEXT = 'text' 31 | 32 | 33 | @dataclass 34 | class Feedback: 35 | vid: str = '' # required. 36 | good: int = 0 # optional; always >= 0. 37 | bad: int = 0 # optional; always >= 0. 38 | 39 | 40 | @unique 41 | class State(StrEnum): 42 | NOTHING = 'nothing' 43 | DOING = 'doing' 44 | DONE = 'done' 45 | 46 | 47 | @dataclass 48 | class TimedText: 49 | start: float = 0 # required; in seconds. 50 | duration: float = 0 # required; in seconds. 51 | lang: str = 'en' # required; language code. 52 | text: str = '' # required. 53 | 54 | 55 | @dataclass 56 | class Translation: 57 | vid: str = '' # required. 58 | cid: str = '' # required. 59 | lang: str = '' # required; language code. 60 | chapter: str = '' # required. 61 | summary: str = '' # required. 62 | 63 | 64 | @dataclass 65 | class User: 66 | uid: str = '' # required. 67 | is_deleted: bool = False # optional. 68 | -------------------------------------------------------------------------------- /database/feedback.py: -------------------------------------------------------------------------------- 1 | from sys import maxsize 2 | from typing import Optional 3 | 4 | from database.data import Feedback 5 | from database.sqlite import commit, fetchall, sqlescape 6 | 7 | _TABLE = 'feedback' 8 | _COLUMN_VID = 'vid' 9 | _COLUMN_GOOD = 'good' 10 | _COLUMN_BAD = 'bad' 11 | _COLUMN_CREATE_TIMESTAMP = 'create_timestamp' 12 | _COLUMN_UPDATE_TIMESTAMP = 'update_timestamp' 13 | 14 | 15 | def create_feedback_table(): 16 | commit(f''' 17 | CREATE TABLE IF NOT EXISTS {_TABLE} ( 18 | {_COLUMN_VID} TEXT NOT NULL PRIMARY KEY, 19 | {_COLUMN_GOOD} INTEGER NOT NULL DEFAULT 0, 20 | {_COLUMN_BAD} INTEGER NOT NULL DEFAULT 0, 21 | {_COLUMN_CREATE_TIMESTAMP} INTEGER NOT NULL DEFAULT 0, 22 | {_COLUMN_UPDATE_TIMESTAMP} INTEGER NOT NULL DEFAULT 0 23 | ) 24 | ''') 25 | 26 | 27 | def find_feedback(vid: str) -> Optional[Feedback]: 28 | res = fetchall(f''' 29 | SELECT 30 | {_COLUMN_VID}, 31 | {_COLUMN_GOOD}, 32 | {_COLUMN_BAD} 33 | FROM {_TABLE} 34 | WHERE {_COLUMN_VID} = '{sqlescape(vid)}' 35 | LIMIT 1 36 | ''') 37 | 38 | if not res: 39 | return None 40 | 41 | res = res[0] 42 | return Feedback( 43 | vid=res[0], 44 | good=res[1], 45 | bad=res[2], 46 | ) 47 | 48 | 49 | def insert_or_update_feedback(feedback: Feedback): 50 | if feedback.good < 0: 51 | feedback.good = 0 52 | elif feedback.good >= maxsize: 53 | feedback.good = maxsize 54 | 55 | if feedback.bad < 0: 56 | feedback.bad = 0 57 | elif feedback.bad >= maxsize: 58 | feedback.bad = maxsize 59 | 60 | previous = find_feedback(feedback.vid) 61 | if not previous: 62 | commit(f''' 63 | INSERT INTO {_TABLE} ( 64 | {_COLUMN_VID}, 65 | {_COLUMN_GOOD}, 66 | {_COLUMN_BAD}, 67 | {_COLUMN_CREATE_TIMESTAMP}, 68 | {_COLUMN_UPDATE_TIMESTAMP} 69 | ) VALUES ( 70 | '{sqlescape(feedback.vid)}', 71 | {feedback.good}, 72 | {feedback.bad}, 73 | STRFTIME('%s', 'NOW'), 74 | STRFTIME('%s', 'NOW') 75 | ) 76 | ''') 77 | else: 78 | commit(f''' 79 | UPDATE {_TABLE} 80 | SET {_COLUMN_GOOD} = {feedback.good}, 81 | {_COLUMN_BAD} = {feedback.good}, 82 | {_COLUMN_UPDATE_TIMESTAMP} = STRFTIME('%s', 'NOW') 83 | WHERE {_COLUMN_VID} = '{sqlescape(feedback.vid)}' 84 | ''') 85 | 86 | 87 | def delete_feedback(vid: str): 88 | commit(f''' 89 | DELETE FROM {_TABLE} 90 | WHERE {_COLUMN_VID} = '{sqlescape(vid)}' 91 | ''') 92 | -------------------------------------------------------------------------------- /database/sqlite.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | 3 | from os import path 4 | from typing import Any 5 | 6 | # https://stackoverflow.com/a/9613153 7 | # 8 | # What if I don't close the database connection in Python SQLite? 9 | # 10 | # In answer to the specific question of what happens if you do not close a SQLite database, 11 | # the answer is quite simple and applies to using SQLite in any programming language. 12 | # When the connection is closed explicitly by code or implicitly by program exit then any outstanding transaction is rolled back. 13 | # (The rollback is actually done by the next program to open the database.) 14 | # If there is no outstanding transaction open then nothing happens. 15 | # 16 | # This means you do not need to worry too much about always closing the database before process exit, 17 | # and that you should pay attention to transactions making sure to start them and commit at appropriate points. 18 | db_connection = sqlite3.connect(path.join(path.dirname(__file__), 'bys.db')) 19 | 20 | 21 | def commit(sql: str): 22 | cursor = db_connection.cursor() 23 | try: 24 | cursor.execute(sql) 25 | db_connection.commit() 26 | finally: 27 | cursor.close() 28 | 29 | 30 | def fetchall(sql: str) -> list[Any]: 31 | cursor = db_connection.cursor() 32 | try: 33 | cursor.execute(sql) 34 | res = cursor.fetchall() 35 | finally: 36 | cursor.close() 37 | return res 38 | 39 | 40 | # https://cs.android.com/android/platform/superproject/+/refs/heads/master:frameworks/base/core/java/android/database/DatabaseUtils.java;drc=7346c436e5a11ce08f6a80dcfeb8ef941ca30176;l=512?q=sqlEscapeString 41 | def sqlescape(string: str) -> str: 42 | res = '' 43 | for c in string: 44 | if c == '\'': 45 | res += '\'' 46 | res += c 47 | return res 48 | 49 | 50 | commit('PRAGMA journal_mode=WAL') 51 | -------------------------------------------------------------------------------- /database/translation.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from database.data import Translation 4 | from database.sqlite import commit, fetchall, sqlescape 5 | 6 | _TABLE = 'translation' 7 | _COLUMN_VID = 'vid' 8 | _COLUMN_CID = 'cid' 9 | _COLUMN_LANG = 'lang' 10 | _COLUMN_CHAPTER = 'chapter' 11 | _COLUMN_SUMMARY = 'summary' 12 | _COLUMN_CREATE_TIMESTAMP = 'create_timestamp' 13 | _COLUMN_UPDATE_TIMESTAMP = 'update_timestamp' 14 | 15 | 16 | def create_translation_table(): 17 | commit(f''' 18 | CREATE TABLE IF NOT EXISTS {_TABLE} ( 19 | {_COLUMN_VID} TEXT NOT NULL DEFAULT '', 20 | {_COLUMN_CID} TEXT NOT NULL DEFAULT '', 21 | {_COLUMN_LANG} TEXT NOT NULL DEFAULT '', 22 | {_COLUMN_CHAPTER} TEXT NOT NULL DEFAULT '', 23 | {_COLUMN_SUMMARY} TEXT NOT NULL DEFAULT '', 24 | {_COLUMN_CREATE_TIMESTAMP} INTEGER NOT NULL DEFAULT 0, 25 | {_COLUMN_UPDATE_TIMESTAMP} INTEGER NOT NULL DEFAULT 0 26 | ) 27 | ''') 28 | commit(f''' 29 | CREATE INDEX IF NOT EXISTS idx_{_COLUMN_VID} 30 | ON {_TABLE} ({_COLUMN_VID}) 31 | ''') 32 | commit(f''' 33 | CREATE INDEX IF NOT EXISTS idx_{_COLUMN_CID} 34 | ON {_TABLE} ({_COLUMN_CID}) 35 | ''') 36 | commit(f''' 37 | CREATE INDEX IF NOT EXISTS idx_{_COLUMN_LANG} 38 | ON {_TABLE} ({_COLUMN_LANG}) 39 | ''') 40 | commit(f''' 41 | CREATE INDEX IF NOT EXISTS idx_{_COLUMN_CREATE_TIMESTAMP} 42 | ON {_TABLE} ({_COLUMN_CREATE_TIMESTAMP}) 43 | ''') 44 | commit(f''' 45 | CREATE INDEX IF NOT EXISTS idx_{_COLUMN_UPDATE_TIMESTAMP} 46 | ON {_TABLE} ({_COLUMN_UPDATE_TIMESTAMP}) 47 | ''') 48 | 49 | 50 | def find_translation(vid: str, cid: str, lang: str) -> Optional[Translation]: 51 | res = fetchall(f''' 52 | SELECT 53 | {_COLUMN_VID}, 54 | {_COLUMN_CID}, 55 | {_COLUMN_LANG}, 56 | {_COLUMN_CHAPTER}, 57 | {_COLUMN_SUMMARY} 58 | FROM {_TABLE} 59 | WHERE {_COLUMN_VID} = '{sqlescape(vid)}' 60 | AND {_COLUMN_CID} = '{sqlescape(cid)}' 61 | AND {_COLUMN_LANG} = '{sqlescape(lang)}' 62 | LIMIT 1 63 | ''') 64 | 65 | if not res: 66 | return None 67 | 68 | res = res[0] 69 | return Translation( 70 | vid=res[0], 71 | cid=res[1], 72 | lang=res[2], 73 | chapter=res[3], 74 | summary=res[4], 75 | ) 76 | 77 | 78 | def insert_or_update_translation(translation: Translation): 79 | previous = find_translation( 80 | vid=translation.vid, 81 | cid=translation.cid, 82 | lang=translation.lang, 83 | ) 84 | if not previous: 85 | commit(f''' 86 | INSERT INTO {_TABLE} ( 87 | {_COLUMN_VID}, 88 | {_COLUMN_CID}, 89 | {_COLUMN_LANG}, 90 | {_COLUMN_CHAPTER}, 91 | {_COLUMN_SUMMARY}, 92 | {_COLUMN_CREATE_TIMESTAMP}, 93 | {_COLUMN_UPDATE_TIMESTAMP} 94 | ) VALUES ( 95 | '{sqlescape(translation.vid)}', 96 | '{sqlescape(translation.cid)}', 97 | '{sqlescape(translation.lang)}', 98 | '{sqlescape(translation.chapter)}', 99 | '{sqlescape(translation.summary)}', 100 | STRFTIME('%s', 'NOW'), 101 | STRFTIME('%s', 'NOW') 102 | ) 103 | ''') 104 | else: 105 | commit(f''' 106 | UPDATE {_TABLE} 107 | SET {_COLUMN_CHAPTER} = '{sqlescape(translation.chapter)}', 108 | {_COLUMN_SUMMARY} = '{sqlescape(translation.summary)}', 109 | {_COLUMN_UPDATE_TIMESTAMP} = STRFTIME('%s', 'NOW') 110 | WHERE {_COLUMN_VID} = '{sqlescape(translation.vid)}' 111 | AND {_COLUMN_CID} = '{sqlescape(translation.cid)}' 112 | AND {_COLUMN_LANG} = '{sqlescape(translation.lang)}' 113 | ''') 114 | 115 | 116 | def delete_translation(vid: str): 117 | commit(f''' 118 | DELETE FROM {_TABLE} 119 | WHERE {_COLUMN_VID} = '{sqlescape(vid)}' 120 | ''') 121 | -------------------------------------------------------------------------------- /database/user.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from database.data import User 4 | from database.sqlite import commit, fetchall, sqlescape 5 | 6 | 7 | _TABLE = 'user' 8 | _COLUMN_UID = 'uid' 9 | _COLUMN_IS_DELETED = 'is_deleted' 10 | _COLUMN_CREATE_TIMESTAMP = 'create_timestamp' 11 | _COLUMN_UPDATE_TIMESTAMP = 'update_timestamp' 12 | 13 | 14 | def create_user_table(): 15 | commit(f''' 16 | CREATE TABLE IF NOT EXISTS {_TABLE} ( 17 | {_COLUMN_UID} TEXT NOT NULL PRIMARY KEY, 18 | {_COLUMN_IS_DELETED} INTEGER NOT NULL DEFAULT 0, 19 | {_COLUMN_CREATE_TIMESTAMP} INTEGER NOT NULL DEFAULT 0, 20 | {_COLUMN_UPDATE_TIMESTAMP} INTEGER NOT NULL DEFAULT 0 21 | ) 22 | ''') 23 | commit(f''' 24 | CREATE INDEX IF NOT EXISTS idx_{_COLUMN_CREATE_TIMESTAMP} 25 | ON {_TABLE} ({_COLUMN_CREATE_TIMESTAMP}) 26 | ''') 27 | commit(f''' 28 | CREATE INDEX IF NOT EXISTS idx_{_COLUMN_UPDATE_TIMESTAMP} 29 | ON {_TABLE} ({_COLUMN_UPDATE_TIMESTAMP}) 30 | ''') 31 | 32 | 33 | def find_user(uid: str) -> Optional[User]: 34 | res = fetchall(f''' 35 | SELECT 36 | {_COLUMN_UID}, 37 | {_COLUMN_IS_DELETED} 38 | FROM {_TABLE} 39 | WHERE {_COLUMN_UID} = '{sqlescape(uid)}' 40 | LIMIT 1 41 | ''') 42 | 43 | if not res: 44 | return None 45 | 46 | res = res[0] 47 | return User( 48 | uid=res[0], 49 | is_deleted=bool(res[1]), 50 | ) 51 | 52 | 53 | def insert_or_update_user(user: User): 54 | previous = find_user(user.uid) 55 | if not previous: 56 | commit(f''' 57 | INSERT INTO {_TABLE} ( 58 | {_COLUMN_UID}, 59 | {_COLUMN_IS_DELETED}, 60 | {_COLUMN_CREATE_TIMESTAMP}, 61 | {_COLUMN_UPDATE_TIMESTAMP} 62 | ) VALUES ( 63 | '{sqlescape(user.uid)}', 64 | {int(user.is_deleted)}, 65 | STRFTIME('%s', 'NOW'), 66 | STRFTIME('%s', 'NOW') 67 | ) 68 | ''') 69 | else: 70 | commit(f''' 71 | UPDATE {_TABLE} 72 | SET {_COLUMN_IS_DELETED} = {int(user.is_deleted)}, 73 | {_COLUMN_UPDATE_TIMESTAMP} = STRFTIME('%s', 'NOW') 74 | WHERE {_COLUMN_UID} = '{sqlescape(user.uid)}' 75 | ''') 76 | -------------------------------------------------------------------------------- /logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | _fmt = '[%(asctime)s] [%(process)d] [%(levelname)s] [%(module)s] %(message)s' 4 | _datefmt = '%Y-%m-%d %H:%M:%S %z' 5 | _handler = logging.StreamHandler() 6 | _handler.setFormatter(logging.Formatter(fmt=_fmt, datefmt=_datefmt)) 7 | 8 | logger = logging.getLogger() 9 | logger.addHandler(_handler) 10 | logger.setLevel(logging.INFO) 11 | -------------------------------------------------------------------------------- /openai.py: -------------------------------------------------------------------------------- 1 | import httpx 2 | import logging 3 | import tiktoken 4 | 5 | from dataclasses import dataclass, asdict 6 | from enum import IntEnum, unique 7 | from quart import abort 8 | from strenum import StrEnum 9 | from tenacity import \ 10 | after_log, \ 11 | retry, \ 12 | retry_if_exception_type, \ 13 | stop_after_attempt, \ 14 | wait_fixed 15 | from werkzeug.exceptions import \ 16 | BadGateway, \ 17 | ServiceUnavailable, \ 18 | TooManyRequests 19 | 20 | from constants import APPLICATION_JSON, USER_AGENT 21 | from logger import logger 22 | from rds import rds, KEY_OPENAI_API_KEY 23 | 24 | 25 | # https://platform.openai.com/docs/models/overview 26 | @unique 27 | class Model(StrEnum): 28 | GPT_3_5_TURBO = 'gpt-3.5-turbo' 29 | GPT_3_5_TURBO_16K = 'gpt-3.5-turbo-16k' 30 | GPT_4 = 'gpt-4' 31 | GPT_4_32K = 'gpt-4-32k' 32 | 33 | 34 | @unique 35 | class TokenLimit(IntEnum): 36 | GPT_3_5_TURBO = 4096 37 | GPT_3_5_TURBO_16K = 16384 38 | GPT_4 = 8192 39 | GPT_4_32K = 32768 40 | 41 | 42 | @unique 43 | class Role(StrEnum): 44 | SYSTEM = 'system' 45 | ASSISTANT = 'assistant' 46 | USER = 'user' 47 | 48 | 49 | @dataclass 50 | class Message: 51 | role: str = '' # required. 52 | content: str = '' # required. 53 | 54 | 55 | # https://platform.openai.com/docs/api-reference/chat/create 56 | _CHAT_API_URL = 'https://api.openai.com/v1/chat/completions' 57 | _encoding_for_chat = tiktoken.get_encoding('cl100k_base') 58 | 59 | 60 | def build_message(role: Role, content: str) -> Message: 61 | return Message(role=role.value, content=content.strip()) 62 | 63 | 64 | # https://platform.openai.com/docs/guides/chat/introduction 65 | def count_tokens(messages: list[Message]) -> int: 66 | tokens_count = 0 67 | 68 | for message in messages: 69 | # Every message follows "{role/name}\n{content}\n". 70 | tokens_count += 4 71 | 72 | for key, value in asdict(message).items(): 73 | tokens_count += len(_encoding_for_chat.encode(value)) 74 | 75 | # If there's a "name", the "role" is omitted. 76 | if key == 'name': 77 | # "role" is always required and always 1 token. 78 | tokens_count += -1 79 | 80 | # Every reply is primed with "assistant". 81 | tokens_count += 2 82 | 83 | return tokens_count 84 | 85 | 86 | # https://platform.openai.com/docs/api-reference/chat/create 87 | @retry( 88 | retry=retry_if_exception_type(( 89 | httpx.ConnectError, 90 | BadGateway, 91 | ServiceUnavailable, 92 | TooManyRequests, 93 | )), 94 | wait=wait_fixed(1), # wait 1 second between retries. 95 | stop=stop_after_attempt(5), # stopping after 5 attempts. 96 | after=after_log(logger, logging.INFO), 97 | ) 98 | async def chat( 99 | messages: list[Message], 100 | model: Model = Model.GPT_3_5_TURBO, 101 | top_p: float = 0.8, # [0, 1] 102 | timeout: int = 10, 103 | api_key: str = '', 104 | ) -> dict: 105 | if not api_key: 106 | api_key = rds.get(KEY_OPENAI_API_KEY).decode() 107 | if not api_key: 108 | abort(500, f'"{KEY_OPENAI_API_KEY}" not exists') 109 | 110 | headers = { 111 | 'User-Agent': USER_AGENT, 112 | 'Content-Type': APPLICATION_JSON, 113 | 'Authorization': f'Bearer {api_key}', 114 | } 115 | 116 | body = { 117 | 'messages': list(map(lambda m: asdict(m), messages)), 118 | 'model': model.value, 119 | 'top_p': top_p, 120 | } 121 | 122 | transport = httpx.AsyncHTTPTransport(retries=2) 123 | client = httpx.AsyncClient(transport=transport) 124 | 125 | try: 126 | response = await client.post( 127 | url=_CHAT_API_URL, 128 | headers=headers, 129 | json=body, 130 | follow_redirects=True, 131 | timeout=timeout, 132 | ) 133 | finally: 134 | await client.aclose() 135 | 136 | if response.status_code not in range(200, 400): 137 | abort(response.status_code, response.text) 138 | 139 | # Automatically .aclose() if the response body is read to completion. 140 | return response.json() 141 | 142 | 143 | def get_content(body: dict) -> str: 144 | return body['choices'][0]['message']['content'] 145 | -------------------------------------------------------------------------------- /pm2.json: -------------------------------------------------------------------------------- 1 | { 2 | "apps": [ 3 | { 4 | "name": "better-youtube-summary-app", 5 | "script": "python3 -m pipenv run hypercorn app:app", 6 | "exec_mode": "fork", 7 | "kill_timeout": 5000, 8 | "listen_timeout": 10000, 9 | "max_memory_restart": "256M", 10 | "watch": false 11 | }, 12 | { 13 | "name": "better-youtube-summary-arq", 14 | "script": "python3 -m pipenv run arq app.WorkerSettings", 15 | "exec_mode": "fork", 16 | "kill_timeout": 5000, 17 | "listen_timeout": 10000, 18 | "max_memory_restart": "768M", 19 | "watch": false 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /prompt.py: -------------------------------------------------------------------------------- 1 | from openai import Message, Role, TokenLimit, build_message 2 | 3 | # For 5 mins video such as https://www.youtube.com/watch?v=tCBknJLD4qY, 4 | # or 10 mins video such as https://www.youtube.com/watch?v=QKOd8TDptt0. 5 | GENERATE_MULTI_CHAPTERS_TOKEN_LIMIT_FOR_4K = TokenLimit.GPT_3_5_TURBO - 512 # nopep8, 3584. 6 | # For more than 15 mins video such as https://www.youtube.com/watch?v=PhFwDJCEhBg. 7 | GENERATE_MULTI_CHAPTERS_TOKEN_LIMIT_FOR_16K = TokenLimit.GPT_3_5_TURBO_16K - 2048 # nopep8, 14336. 8 | 9 | # Looks like use the word "outline" is better than the work "chapter". 10 | _GENERATE_MULTI_CHAPTERS_SYSTEM_PROMPT = ''' 11 | Given the following video subtitles represented as a JSON array as shown below: 12 | 13 | ```json 14 | [ 15 | {{ 16 | "start": int field, the subtitle start time in seconds. 17 | "text": string field, the subtitle text itself. 18 | }} 19 | ] 20 | ``` 21 | 22 | Please generate the subtitles' outlines from top to bottom, 23 | and extract an useful information from each outline context; 24 | each useful information should end with a period; 25 | exclude the introduction at the beginning and the conclusion at the end; 26 | exclude text like "[Music]", "[Applause]", "[Laughter]" and so on. 27 | 28 | Return a JSON array as shown below: 29 | 30 | ```json 31 | [ 32 | {{ 33 | "outline": string field, a brief outline title in language "{lang}". 34 | "information": string field, an useful information in the outline context in language "{lang}". 35 | "start": int field, the start time of the outline in seconds. 36 | "timestamp": string field, the start time of the outline in "HH:mm:ss" format. 37 | }} 38 | ] 39 | ``` 40 | 41 | Please output JSON only. 42 | Do not output any redundant explanation. 43 | ''' 44 | 45 | _GENERATE_MULTI_CHAPTERS_USER_MESSAGE_FOR_16K = ''' 46 | [ 47 | {{"start": 0, "text": "Hi everyone, this is Chef Wang. Today, I will show everyone how to make"}}, 48 | {{"start": 3, "text": "Egg Fried Rice."}}, 49 | {{"start": 4, "text": "First, we'll need cold cooked rice (can be leftover)."}}, 50 | {{"start": 8, "text": "Crack two eggs into a bowl."}}, 51 | {{"start": 14, "text": "Separate yolk from whites."}}, 52 | {{"start": 16, "text": "Beat the yolk and set aside."}}, 53 | {{"start": 19, "text": "Next we will prepare the mix-ins."}}, 54 | {{"start": 22, "text": "Chop the kernels off the corn."}}, 55 | {{"start": 24, "text": "The corn adds sweetness"}}, 56 | {{"start": 27, "text": "The following ingredients are optional."}}, 57 | {{"start": 30, "text": "Dice up some bacon."}}, 58 | {{"start": 33, "text": "The bacon will help season the dish and add umami."}}, 59 | {{"start": 38, "text": "Dice a small knob of lapcheong (chinese sausage)."}}, 60 | {{"start": 40, "text": "Like the bacon, it adds salt and savoriness."}}, 61 | {{"start": 44, "text": "Chop up 2 shiitake mushrooms."}}, 62 | {{"start": 47, "text": "The mushroom adds umami, and replaces msg in the seasoning."}}, 63 | {{"start": 52, "text": "Now let's start to cook."}}, 64 | {{"start": 56, "text": "First, heat up the wok."}}, 65 | {{"start": 59, "text": "Add enough oil to coat."}}, 66 | {{"start": 63, "text": "Remove the heated oil, then add cooking oil."}}, 67 | {{"start": 67, "text": "Add the whites, and stirfry until cooked."}}, 68 | {{"start": 72, "text": "When the egg whites are cooked, remove from wok and set aside."}}, 69 | {{"start": 77, "text": "Add a little vegetable, cook the yolks until fragrant."}}, 70 | {{"start": 89, "text": "Then add the prepared bacon, sausage, and corn."}}, 71 | {{"start": 97, "text": "Lower the heat to medium-low to allow ingredients to cook through."}}, 72 | {{"start": 104, "text": "Then, add the egg whites back."}}, 73 | {{"start": 107, "text": "Toss to cook everything evenly."}}, 74 | {{"start": 110, "text": "Then add the prepared cold rice."}}, 75 | {{"start": 114, "text": "Add the minced mushrooms." }}, 76 | {{"start": 118, "text": "Turn the heat very low and toss to stir fry the rice for five minutes."}}, 77 | {{"start": 121, "text": "This allows the rice to absorb seasoning from all our add-ins."}}, 78 | {{"start": 125, "text": "You must stir fry until the wok begins to smoke (wok hei)"}}, 79 | {{"start": 131, "text": "At this point (good wok hei),"}}, 80 | {{"start": 132, "text": "drizzle in a small amount of soy sauce from the edges of the wok."}}, 81 | {{"start": 135, "text": "Crank heat to high, and toss a few more times."}}, 82 | {{"start": 138, "text": "Add some chopped scallion, toss to mix, and it's ready to plate."}}, 83 | {{"start": 151, "text": "A delicious plate of homestyle fried rice is now finished."}}, 84 | {{"start": 155, "text": "Technical summary:"}}, 85 | {{"start": 157, "text": "1: You can change the mix-ins according to your tastes."}}, 86 | {{"start": 161, "text": "2: The rice must be cold, or it will clump into a mushy ball."}}, 87 | {{"start": 164, "text": "3: This recipe has bacon and sausage, so we did not add more salt to avoid over salting."}}, 88 | {{"start": 170, "text": "4: Wok hei is all about the heat of the wok and the ingredients (??)."}}, 89 | {{"start": 175, "text": "There will be a follow up video to go in more depth."}}, 90 | {{"start": 179, "text": "This concludes the technical summary for \"Homestyle egg fried rice\""}} 91 | ] 92 | ''' 93 | 94 | _GENERATE_MULTI_CHAPTERS_ASSISTANT_MESSAGE_FOR_16K = ''' 95 | [ 96 | {{ 97 | "outline": "Ingredients preparation", 98 | "information": "Chef Wang explains the ingredients needed for the dish, including cold cooked rice, eggs, corn, bacon, lapcheong, and shiitake mushrooms.", 99 | "start": 4, 100 | "timestamp": "00:00:04" 101 | }}, 102 | {{ 103 | "outline": "Cooking process", 104 | "information": "Chef Wang demonstrates the cooking process, including heating up the wok, cooking the egg whites and yolks, adding the mix-ins, and stir-frying the rice.", 105 | "start": 52, 106 | "timestamp": "00:00:52" 107 | }}, 108 | {{ 109 | "outline": "Seasoning", 110 | "information": "Chef Wang explains the importance of stir-frying until the wok begins to smoke (wok hei) and adding soy sauce and scallions for seasoning.", 111 | "start": 125, 112 | "timestamp": "00:02:05" 113 | }}, 114 | {{ 115 | "outline": "Technical summary", 116 | "information": "Chef Wang provides some technical tips for making the dish, including changing the mix-ins according to taste, using cold rice, avoiding over-salting, and achieving wok hei.", 117 | "start": 155, 118 | "timestamp": "00:02:35" 119 | }} 120 | ] 121 | ''' 122 | 123 | # For more than 30 mins video such as https://www.youtube.com/watch?v=WRLVrfIBS1k. 124 | GENERATE_ONE_CHAPTER_TOKEN_LIMIT = TokenLimit.GPT_3_5_TURBO - 160 # nopep8, 3936. 125 | # Looks like use the word "outline" is better than the work "chapter". 126 | GENERATE_ONE_CHAPTER_SYSTEM_PROMPT = ''' 127 | Given a part of video subtitles JSON array as shown below: 128 | 129 | ```json 130 | [ 131 | {{ 132 | "index": int field, the subtitle line index. 133 | "start": int field, the subtitle start time in seconds. 134 | "text": string field, the subtitle text itself. 135 | }} 136 | ] 137 | ``` 138 | 139 | Your job is trying to generate the subtitles' outline with follow steps: 140 | 141 | 1. Extract an useful information as the outline context, 142 | 2. exclude out-of-context parts and irrelevant parts, 143 | 3. exclude text like "[Music]", "[Applause]", "[Laughter]" and so on, 144 | 4. summarize the useful information to one-word as the outline title. 145 | 146 | Please return a JSON object as shown below: 147 | 148 | ```json 149 | {{ 150 | "end_at": int field, the outline context end at which subtitle index. 151 | "start": int field, the start time of the outline context in seconds, must >= {start_time}. 152 | "timestamp": string field, the start time of the outline context in "HH:mm:ss" format. 153 | "outline": string field, the outline title in language "{lang}". 154 | }} 155 | ``` 156 | 157 | Please output JSON only. 158 | Do not output any redundant explanation. 159 | ''' 160 | 161 | # https://github.com/hwchase17/langchain/blob/master/langchain/chains/summarize/refine_prompts.py#L21 162 | SUMMARIZE_FIRST_CHAPTER_TOKEN_LIMIT = TokenLimit.GPT_3_5_TURBO - 512 # nopep8, 3584. 163 | SUMMARIZE_FIRST_CHAPTER_SYSTEM_PROMPT = ''' 164 | Given a part of video subtitles about "{chapter}". 165 | Please summarize and list the most important points of the subtitles. 166 | 167 | The subtitles consists of many lines. 168 | The format of each line is like `[text...]`, for example `[hello, world]`. 169 | 170 | The output format should be a markdown bullet list, and each bullet point should end with a period. 171 | The output language should be "{lang}" in ISO 639-1. 172 | 173 | Please exclude line like "[Music]", "[Applause]", "[Laughter]" and so on. 174 | Please merge similar viewpoints before the final output. 175 | Please keep the output clear and accurate. 176 | 177 | Do not output any redundant or irrelevant points. 178 | Do not output any redundant explanation or information. 179 | ''' 180 | 181 | # https://github.com/hwchase17/langchain/blob/master/langchain/chains/summarize/refine_prompts.py#L4 182 | SUMMARIZE_NEXT_CHAPTER_TOKEN_LIMIT = TokenLimit.GPT_3_5_TURBO * 5 / 8 # nopep8, 2560. 183 | SUMMARIZE_NEXT_CHAPTER_SYSTEM_PROMPT = ''' 184 | We have provided an existing bullet list summary up to a certain point: 185 | 186 | ``` 187 | {summary} 188 | ``` 189 | 190 | We have the opportunity to refine the existing summary (only if needed) with some more content. 191 | 192 | The content is a part of video subtitles about "{chapter}", consists of many lines. 193 | The format of each line is like `[text...]`, for example `[hello, world]`. 194 | 195 | Please refine the existing bullet list summary (only if needed) with the given content. 196 | If the the given content isn't useful or doesn't make sense, don't refine the the existing summary. 197 | 198 | The output format should be a markdown bullet list, and each bullet point should end with a period. 199 | The output language should be "{lang}" in BCP 47. 200 | 201 | Please exclude line like "[Music]", "[Applause]", "[Laughter]" and so on. 202 | Please merge similar viewpoints before the final output. 203 | Please keep the output clear and accurate. 204 | 205 | Do not output any redundant or irrelevant points. 206 | Do not output any redundant explanation or information. 207 | ''' 208 | 209 | 210 | def generate_multi_chapters_example_messages_for_4k(lang: str) -> list[Message]: 211 | system_prompt = _GENERATE_MULTI_CHAPTERS_SYSTEM_PROMPT.format(lang=lang) 212 | return [build_message(Role.SYSTEM, system_prompt)] 213 | 214 | 215 | def generate_multi_chapters_example_messages_for_16k(lang: str) -> list[Message]: 216 | system_prompt = _GENERATE_MULTI_CHAPTERS_SYSTEM_PROMPT.format(lang=lang) 217 | system_message = build_message(Role.SYSTEM, system_prompt) 218 | user_message = build_message(Role.USER, _GENERATE_MULTI_CHAPTERS_USER_MESSAGE_FOR_16K) # nopep8. 219 | assistant_message = build_message(Role.ASSISTANT, _GENERATE_MULTI_CHAPTERS_ASSISTANT_MESSAGE_FOR_16K) # nopep8. 220 | return [system_message, user_message, assistant_message] 221 | -------------------------------------------------------------------------------- /rds.py: -------------------------------------------------------------------------------- 1 | import redis 2 | 3 | # https://github.com/aio-libs/aioredis-py 4 | from redis import asyncio as aioredis 5 | 6 | KEY_OPENAI_API_KEY = 'openai_api_key' # string. 7 | 8 | # Default host and port. 9 | rds = redis.from_url('redis://localhost:6379') 10 | ards = aioredis.from_url('redis://localhost:6379') 11 | -------------------------------------------------------------------------------- /sse.py: -------------------------------------------------------------------------------- 1 | import async_timeout 2 | import json 3 | 4 | from dataclasses import dataclass, asdict, field 5 | from enum import unique 6 | 7 | from strenum import StrEnum 8 | 9 | from logger import logger 10 | from rds import ards 11 | 12 | 13 | @unique 14 | class SseEvent(StrEnum): 15 | SUMMARY = 'summary' 16 | CLOSE = 'close' 17 | 18 | 19 | @dataclass 20 | class SseMessage: 21 | event: str = '' # required. 22 | data: dict or list[dict] = field(default_factory=dict or list[dict]) # nopep8; required. 23 | 24 | def __str__(self) -> str: 25 | data_str = json.dumps(self.data) 26 | lines = [f'data: {line}' for line in data_str.splitlines()] 27 | lines.insert(0, f'event: {self.event}') 28 | return '\n'.join(lines) + '\n\n' 29 | 30 | 31 | async def sse_publish(channel: str, event: SseEvent, data: dict or list[dict] = {}): 32 | message = SseMessage(event=event.value, data=data) 33 | message = json.dumps(asdict(message)) 34 | await ards.publish(channel=channel, message=message) 35 | 36 | 37 | # https://aioredis.readthedocs.io/en/latest/getting-started/#pubsub-mode 38 | async def sse_subscribe(channel: str): 39 | pubsub = ards.pubsub() 40 | await pubsub.subscribe(channel) 41 | logger.info(f'sse_subscribe, channel={channel}') 42 | 43 | try: 44 | while True: 45 | async with async_timeout.timeout(300): # 5 mins. 46 | obj = await pubsub.get_message(ignore_subscribe_messages=True) 47 | if isinstance(obj, dict): 48 | message = SseMessage(**json.loads(obj['data'])) 49 | yield str(message) 50 | 51 | if message.event == SseEvent.CLOSE: 52 | logger.info(f'sse_subscribe, on close, channel={channel}') # nopep8. 53 | break # while. 54 | finally: 55 | await sse_unsubscribe(channel) 56 | 57 | 58 | async def sse_unsubscribe(channel: str): 59 | try: 60 | await ards.pubsub().unsubscribe(channel) 61 | logger.info(f'sse_unsubscribe, channel={channel}') 62 | except Exception: 63 | logger.exception(f'sse_unsubscribe, channel={channel}') 64 | -------------------------------------------------------------------------------- /summary.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import json 3 | 4 | from dataclasses import asdict 5 | from sys import maxsize 6 | from uuid import uuid4 7 | 8 | from quart import abort 9 | from youtube_transcript_api import YouTubeTranscriptApi 10 | 11 | from database.data import \ 12 | Chapter, \ 13 | ChapterSlicer, \ 14 | ChapterStyle, \ 15 | State, \ 16 | TimedText 17 | from database.feedback import find_feedback 18 | from logger import logger 19 | from openai import Model, Role, \ 20 | build_message, \ 21 | chat, \ 22 | count_tokens, \ 23 | get_content 24 | from prompt import \ 25 | GENERATE_MULTI_CHAPTERS_TOKEN_LIMIT_FOR_4K, \ 26 | GENERATE_MULTI_CHAPTERS_TOKEN_LIMIT_FOR_16K, \ 27 | GENERATE_ONE_CHAPTER_SYSTEM_PROMPT, \ 28 | GENERATE_ONE_CHAPTER_TOKEN_LIMIT, \ 29 | SUMMARIZE_FIRST_CHAPTER_SYSTEM_PROMPT, \ 30 | SUMMARIZE_FIRST_CHAPTER_TOKEN_LIMIT, \ 31 | SUMMARIZE_NEXT_CHAPTER_SYSTEM_PROMPT, \ 32 | SUMMARIZE_NEXT_CHAPTER_TOKEN_LIMIT, \ 33 | generate_multi_chapters_example_messages_for_4k, \ 34 | generate_multi_chapters_example_messages_for_16k 35 | from rds import rds 36 | from sse import SseEvent, sse_publish 37 | 38 | SUMMARIZING_RDS_KEY_EX = 300 # 5 mins. 39 | NO_TRANSCRIPT_RDS_KEY_EX = 8 * 60 * 60 # 8 hours. 40 | 41 | 42 | def build_summary_channel(vid: str) -> str: 43 | return f'summary_{vid}' 44 | 45 | 46 | def build_summary_response(state: State, chapters: list[Chapter] = []) -> dict: 47 | chapters = list(map(lambda c: asdict(c), chapters)) 48 | return { 49 | 'state': state.value, 50 | 'chapters': chapters, 51 | } 52 | 53 | 54 | def build_summarizing_rds_key(vid: str) -> str: 55 | return f'summarizing_{vid}' 56 | 57 | 58 | def build_no_transcript_rds_key(vid: str) -> str: 59 | return f'no_transcript_{vid}' 60 | 61 | 62 | async def do_if_found_chapters_in_database(vid: str, chapters: list[Chapter]): 63 | rds.delete(build_no_transcript_rds_key(vid)) 64 | rds.delete(build_summarizing_rds_key(vid)) 65 | channel = build_summary_channel(vid) 66 | data = build_summary_response(State.DONE, chapters) 67 | await sse_publish(channel=channel, event=SseEvent.SUMMARY, data=data) 68 | await sse_publish(channel=channel, event=SseEvent.CLOSE) 69 | 70 | 71 | def need_to_resummarize(vid: str, chapters: list[Chapter] = []) -> bool: 72 | for c in chapters: 73 | if (not c.summary) or len(c.summary) <= 0: 74 | return True 75 | 76 | feedback = find_feedback(vid) 77 | if not feedback: 78 | return False 79 | 80 | good = feedback.good if feedback.good > 0 else 1 81 | bad = feedback.bad if feedback.bad > 0 else 1 82 | 83 | # DO NOTHING if total less then 10. 84 | if good + bad < 10: 85 | return False 86 | 87 | # Need to resummarize if bad percent >= 20% 88 | return bad / (good + bad) >= 0.2 89 | 90 | 91 | # NoTranscriptFound, TranscriptsDisabled... 92 | def parse_timed_texts_and_lang(vid: str) -> tuple[list[TimedText], str]: 93 | timed_texts: list[TimedText] = [] 94 | 95 | # https://en.wikipedia.org/wiki/Languages_used_on_the_Internet#Content_languages_on_YouTube 96 | codes = [ 97 | 'en', # English. 98 | 'es', # Spanish. 99 | 'pt', # Portuguese. 100 | 'hi', # Hindi. 101 | 'ko', # Korean. 102 | 'zh-Hans', # Chinese (Simplified). 103 | 'zh-Hant', # Chinese (Traditional). 104 | 'zh-CN', # Chinese (China). 105 | 'zh-HK', # Chinese (Hong Kong). 106 | 'zh-TW', # Chinese (Taiwan). 107 | 'zh', # Chinese. 108 | 'ar', # Arabic. 109 | 'id', # Indonesian. 110 | 'fr', # French. 111 | 'ja', # Japanese. 112 | 'ru', # Russian. 113 | 'de', # German. 114 | ] 115 | 116 | transcript_list = YouTubeTranscriptApi.list_transcripts(vid) 117 | 118 | try: 119 | transcript = transcript_list.find_manually_created_transcript(codes) 120 | except Exception: # NoTranscriptFound. 121 | # logger.exception(f'find manually created transcript failed, vid={vid}') 122 | transcript = transcript_list.find_generated_transcript(codes) 123 | 124 | lang = transcript.language_code 125 | array: list[dict] = transcript.fetch() 126 | 127 | for d in array: 128 | timed_texts.append(TimedText( 129 | start=d['start'], 130 | duration=d['duration'], 131 | lang=lang, 132 | text=d['text'], 133 | )) 134 | 135 | return timed_texts, lang 136 | 137 | 138 | async def summarize( 139 | vid: str, 140 | trigger: str, 141 | chapters: list[dict], 142 | timed_texts: list[TimedText], 143 | lang: str, 144 | openai_api_key: str = '', 145 | ) -> tuple[list[Chapter], bool]: 146 | logger.info( 147 | f'summarize, ' 148 | f'vid={vid}, ' 149 | f'len(chapters)={len(chapters)}, ' 150 | f'len(timed_texts)={len(timed_texts)}, ' 151 | f'lang={lang}') 152 | 153 | has_exception = False 154 | chapters: list[Chapter] = _parse_chapters( 155 | vid=vid, 156 | trigger=trigger, 157 | chapters=chapters, 158 | lang=lang, 159 | ) 160 | 161 | if not chapters: 162 | # Use the "outline" and "information" fields if they can be generated in 4k. 163 | chapters = await _generate_multi_chapters( 164 | vid=vid, 165 | trigger=trigger, 166 | timed_texts=timed_texts, 167 | lang=lang, 168 | model=Model.GPT_3_5_TURBO, 169 | openai_api_key=openai_api_key, 170 | ) 171 | if chapters: 172 | await _do_before_return(vid, chapters) 173 | return chapters, has_exception 174 | 175 | # Just use the "outline" field if it can be generated in 16k. 176 | chapters = await _generate_multi_chapters( 177 | vid=vid, 178 | trigger=trigger, 179 | timed_texts=timed_texts, 180 | lang=lang, 181 | model=Model.GPT_3_5_TURBO_16K, 182 | openai_api_key=openai_api_key, 183 | ) 184 | 185 | if not chapters: 186 | chapters = await _generate_chapters_one_by_one( 187 | vid=vid, 188 | trigger=trigger, 189 | timed_texts=timed_texts, 190 | lang=lang, 191 | openai_api_key=openai_api_key, 192 | ) 193 | 194 | if not chapters: 195 | abort(500, f'summarize failed, no chapters, vid={vid}') 196 | else: 197 | await sse_publish( 198 | channel=build_summary_channel(vid), 199 | event=SseEvent.SUMMARY, 200 | data=build_summary_response(State.DOING, chapters), 201 | ) 202 | 203 | tasks = [] 204 | for i, c in enumerate(chapters): 205 | start_time = c.start 206 | end_time = chapters[i + 1].start if i + 1 < len(chapters) else maxsize # nopep8. 207 | texts = _get_timed_texts_in_range( 208 | timed_texts=timed_texts, 209 | start_time=start_time, 210 | end_time=end_time, 211 | ) 212 | tasks.append(_summarize_chapter( 213 | chapter=c, 214 | timed_texts=texts, 215 | lang=lang, 216 | openai_api_key=openai_api_key, 217 | )) 218 | 219 | res = await asyncio.gather(*tasks, return_exceptions=True) 220 | for r in res: 221 | if isinstance(r, Exception): 222 | logger.error(f'summarize, but has exception, vid={vid}, e={r}') 223 | has_exception = True 224 | 225 | await _do_before_return(vid, chapters) 226 | return chapters, has_exception 227 | 228 | 229 | def _parse_chapters( 230 | vid: str, 231 | trigger: str, 232 | chapters: list[dict], 233 | lang: str, 234 | ) -> list[Chapter]: 235 | res: list[Chapter] = [] 236 | 237 | if not chapters: 238 | logger.info(f'parse chapters, but chapters is empty, vid={vid}') 239 | return res 240 | 241 | try: 242 | for c in chapters: 243 | timestamp: str = c['timestamp'] 244 | 245 | seconds: int = 0 246 | array: list[str] = timestamp.split(':') 247 | if len(array) == 2: 248 | seconds = int(array[0]) * 60 + int(array[1]) 249 | elif len(array) == 3: 250 | seconds = int(array[0]) * 60 * 60 + int(array[1]) * 60 + int(array[2]) # nopep8. 251 | 252 | res.append(Chapter( 253 | cid=str(uuid4()), 254 | vid=vid, 255 | trigger=trigger, 256 | slicer=ChapterSlicer.YOUTUBE.value, 257 | style=ChapterStyle.MARKDOWN.value, 258 | start=seconds, 259 | lang=lang, 260 | chapter=c['title'], 261 | )) 262 | except Exception: 263 | logger.exception(f'parse chapters failed, vid={vid}') 264 | return res 265 | 266 | return res 267 | 268 | 269 | # FIXME (Matthew Lee) suppurt stream. 270 | async def _generate_multi_chapters( 271 | vid: str, 272 | trigger: str, 273 | timed_texts: list[TimedText], 274 | lang: str, 275 | model: Model = Model.GPT_3_5_TURBO, 276 | openai_api_key: str = '', 277 | ) -> list[Chapter]: 278 | chapters: list[Chapter] = [] 279 | content: list[dict] = [] 280 | 281 | for t in timed_texts: 282 | text = t.text.strip() 283 | if not text: 284 | continue 285 | content.append({ 286 | 'start': int(t.start), 287 | 'text': text, 288 | }) 289 | 290 | user_message = build_message( 291 | role=Role.USER, 292 | content=json.dumps(content, ensure_ascii=False), 293 | ) 294 | 295 | if model == Model.GPT_3_5_TURBO: 296 | messages = generate_multi_chapters_example_messages_for_4k(lang=lang) 297 | messages.append(user_message) 298 | count = count_tokens(messages) 299 | if count >= GENERATE_MULTI_CHAPTERS_TOKEN_LIMIT_FOR_4K: 300 | logger.info(f'generate multi chapters with 4k, reach token limit, vid={vid}, count={count}') # nopep8. 301 | return chapters 302 | elif model == Model.GPT_3_5_TURBO_16K: 303 | messages = generate_multi_chapters_example_messages_for_16k(lang=lang) 304 | messages.append(user_message) 305 | count = count_tokens(messages) 306 | if count >= GENERATE_MULTI_CHAPTERS_TOKEN_LIMIT_FOR_16K: 307 | logger.info(f'generate multi chapters with 16k, reach token limit, vid={vid}, count={count}') # nopep8. 308 | return chapters 309 | else: 310 | abort(500, f'generate multi chapters with wrong model, model={model}') 311 | 312 | try: 313 | body = await chat( 314 | messages=messages, 315 | model=model, 316 | top_p=0.1, 317 | timeout=90, 318 | api_key=openai_api_key, 319 | ) 320 | 321 | content = get_content(body) 322 | logger.info(f'generate multi chapters, vid={vid}, content=\n{content}') 323 | 324 | # FIXME (Matthew Lee) prompt output as JSON may not work (in the end). 325 | res: list[dict] = json.loads(content) 326 | except Exception: 327 | logger.exception(f'generate multi chapters failed, vid={vid}') 328 | return chapters 329 | 330 | for r in res: 331 | chapter = r.get('outline', '').strip() 332 | information = r.get('information', '').strip() 333 | seconds = r.get('start', -1) 334 | 335 | if chapter and information and seconds >= 0: 336 | chapters.append(Chapter( 337 | cid=str(uuid4()), 338 | vid=vid, 339 | trigger=trigger, 340 | slicer=ChapterSlicer.OPENAI.value, 341 | style=ChapterStyle.TEXT.value, 342 | start=seconds, 343 | lang=lang, 344 | chapter=chapter, 345 | summary=information, 346 | )) 347 | 348 | # FIXME (Matthew Lee) prompt output may not sortd by seconds asc. 349 | return sorted(chapters, key=lambda c: c.start) 350 | 351 | 352 | async def _generate_chapters_one_by_one( 353 | vid: str, 354 | trigger: str, 355 | timed_texts: list[TimedText], 356 | lang: str, 357 | openai_api_key: str = '', 358 | ) -> list[Chapter]: 359 | chapters: list[Chapter] = [] 360 | timed_texts_start = 0 361 | latest_end_at = -1 362 | 363 | while True: 364 | texts = timed_texts[timed_texts_start:] 365 | if not texts: 366 | logger.info(f'generate one chapter, drained, ' 367 | f'vid={vid}, ' 368 | f'len={len(timed_texts)}, ' 369 | f'timed_texts_start={timed_texts_start}') 370 | break # drained. 371 | 372 | system_prompt = GENERATE_ONE_CHAPTER_SYSTEM_PROMPT.format( 373 | start_time=int(texts[0].start), 374 | lang=lang, 375 | ) 376 | system_message = build_message(Role.SYSTEM, system_prompt) 377 | 378 | content: list[dict] = [] 379 | for t in texts: 380 | text = t.text.strip() 381 | if not text: 382 | continue 383 | 384 | temp = content.copy() 385 | temp.append({ 386 | 'index': timed_texts_start, 387 | 'start': int(t.start), 388 | 'text': text, 389 | }) 390 | 391 | user_message = build_message( 392 | role=Role.USER, 393 | content=json.dumps(temp, ensure_ascii=False), 394 | ) 395 | 396 | if count_tokens([system_message, user_message]) < GENERATE_ONE_CHAPTER_TOKEN_LIMIT: 397 | content = temp 398 | timed_texts_start += 1 399 | else: 400 | break # for. 401 | 402 | user_message = build_message( 403 | role=Role.USER, 404 | content=json.dumps(content, ensure_ascii=False), 405 | ) 406 | 407 | logger.info(f'generate one chapter, ' 408 | f'vid={vid}, ' 409 | f'latest_end_at={latest_end_at}, ' 410 | f'timed_texts_start={timed_texts_start}') 411 | 412 | try: 413 | body = await chat( 414 | messages=[system_message, user_message], 415 | model=Model.GPT_3_5_TURBO, 416 | top_p=0.1, 417 | timeout=90, 418 | api_key=openai_api_key, 419 | ) 420 | 421 | content = get_content(body) 422 | logger.info(f'generate one chapter, vid={vid}, content=\n{content}') # nopep8. 423 | 424 | # FIXME (Matthew Lee) prompt output as JSON may not work (in the end). 425 | res: dict = json.loads(content) 426 | except Exception: 427 | logger.exception(f'generate one chapter failed, vid={vid}') 428 | break # drained. 429 | 430 | chapter = res.get('outline', '').strip() 431 | seconds = res.get('start', -1) 432 | end_at = res.get('end_at') 433 | 434 | # Looks like it's the end and meanless, so ignore the chapter. 435 | if type(end_at) is not int: # NoneType. 436 | logger.info(f'generate one chapter, end_at is not int, vid={vid}') 437 | break # drained. 438 | 439 | if chapter and seconds >= 0: 440 | data = Chapter( 441 | cid=str(uuid4()), 442 | vid=vid, 443 | trigger=trigger, 444 | slicer=ChapterSlicer.OPENAI.value, 445 | style=ChapterStyle.MARKDOWN.value, 446 | start=seconds, 447 | lang=lang, 448 | chapter=chapter, 449 | ) 450 | 451 | chapters.append(data) 452 | await sse_publish( 453 | channel=build_summary_channel(vid), 454 | event=SseEvent.SUMMARY, 455 | data=build_summary_response(State.DOING, chapters), 456 | ) 457 | 458 | # Looks like it's the end and meanless, so ignore the chapter. 459 | # if type(end_at) is not int: # NoneType. 460 | # logger.info(f'generate chapters, end_at is not int, vid={vid}') 461 | # break # drained. 462 | 463 | if end_at <= latest_end_at: 464 | logger.warning(f'generate one chapter, avoid infinite loop, vid={vid}') # nopep8. 465 | latest_end_at += 5 # force a different context. 466 | timed_texts_start = latest_end_at 467 | elif end_at > timed_texts_start: 468 | logger.warning(f'generate one chapter, avoid drain early, vid={vid}') # nopep8. 469 | latest_end_at = timed_texts_start 470 | timed_texts_start = latest_end_at + 1 471 | else: 472 | latest_end_at = end_at 473 | timed_texts_start = end_at + 1 474 | 475 | return chapters 476 | 477 | 478 | def _get_timed_texts_in_range(timed_texts: list[TimedText], start_time: int, end_time: int = maxsize) -> list[TimedText]: 479 | res: list[TimedText] = [] 480 | 481 | for t in timed_texts: 482 | if start_time <= t.start and t.start < end_time: 483 | res.append(t) 484 | 485 | return res 486 | 487 | 488 | async def _summarize_chapter( 489 | chapter: Chapter, 490 | timed_texts: list[TimedText], 491 | lang: str, 492 | openai_api_key: str = '', 493 | ): 494 | vid = chapter.vid 495 | summary = '' 496 | summary_start = 0 497 | refined_count = 0 498 | 499 | while True: 500 | texts = timed_texts[summary_start:] 501 | if not texts: 502 | break # drained. 503 | 504 | content = '' 505 | content_has_changed = False 506 | 507 | for t in texts: 508 | lines = content + '\n' + f'[{t.text}]' if content else f'[{t.text}]' # nopep8. 509 | if refined_count <= 0: 510 | system_prompt = SUMMARIZE_FIRST_CHAPTER_SYSTEM_PROMPT.format( 511 | chapter=chapter.chapter, 512 | lang=lang, 513 | ) 514 | else: 515 | system_prompt = SUMMARIZE_NEXT_CHAPTER_SYSTEM_PROMPT.format( 516 | chapter=chapter.chapter, 517 | summary=summary, 518 | lang=lang, 519 | ) 520 | 521 | system_message = build_message(Role.SYSTEM, system_prompt) 522 | user_message = build_message(Role.USER, lines) 523 | token_limit = SUMMARIZE_FIRST_CHAPTER_TOKEN_LIMIT \ 524 | if refined_count <= 0 else SUMMARIZE_NEXT_CHAPTER_TOKEN_LIMIT 525 | 526 | if count_tokens([system_message, user_message]) < token_limit: 527 | content_has_changed = True 528 | content = lines.strip() 529 | summary_start += 1 530 | else: 531 | break # for. 532 | 533 | # FIXME (Matthew Lee) it is possible that content not changed, simply avoid redundant requests. 534 | if not content_has_changed: 535 | logger.warning(f'summarize chapter, but content not changed, vid={vid}') # nopep8. 536 | break 537 | 538 | if refined_count <= 0: 539 | system_prompt = SUMMARIZE_FIRST_CHAPTER_SYSTEM_PROMPT.format( 540 | chapter=chapter.chapter, 541 | lang=lang, 542 | ) 543 | else: 544 | system_prompt = SUMMARIZE_NEXT_CHAPTER_SYSTEM_PROMPT.format( 545 | chapter=chapter.chapter, 546 | summary=summary, 547 | lang=lang, 548 | ) 549 | 550 | system_message = build_message(Role.SYSTEM, system_prompt) 551 | user_message = build_message(Role.USER, content) 552 | body = await chat( 553 | messages=[system_message, user_message], 554 | model=Model.GPT_3_5_TURBO, 555 | top_p=0.1, 556 | timeout=90, 557 | api_key=openai_api_key, 558 | ) 559 | 560 | summary = get_content(body).strip() 561 | chapter.summary = summary # cache even not finished. 562 | refined_count += 1 563 | 564 | chapter.summary = summary.strip() 565 | chapter.refined = refined_count - 1 if refined_count > 0 else 0 566 | 567 | await sse_publish( 568 | channel=build_summary_channel(vid), 569 | event=SseEvent.SUMMARY, 570 | data=build_summary_response(State.DOING, [chapter]), 571 | ) 572 | 573 | 574 | async def _do_before_return(vid: str, chapters: list[Chapter]): 575 | channel = build_summary_channel(vid) 576 | data = build_summary_response(State.DONE, chapters) 577 | await sse_publish(channel=channel, event=SseEvent.SUMMARY, data=data) 578 | await sse_publish(channel=channel, event=SseEvent.CLOSE) 579 | -------------------------------------------------------------------------------- /translation.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from typing import Optional 4 | 5 | from langcodes import Language 6 | from quart import abort 7 | 8 | from database.chapter import find_chapter_by_cid 9 | from database.data import Translation 10 | from database.translation import \ 11 | find_translation, \ 12 | insert_or_update_translation 13 | from logger import logger 14 | from openai import Model, Role, \ 15 | build_message, \ 16 | chat, \ 17 | count_tokens, \ 18 | get_content 19 | 20 | _TRANSLATION_SYSTEM_PROMPT = ''' 21 | Given the following JSON object as shown below: 22 | 23 | ```json 24 | {{ 25 | "chapter": "text...", 26 | "summary": "text..." 27 | }} 28 | ``` 29 | 30 | Translate the "chapter" field and "summary" field to language {lang} in BCP 47, 31 | the translation should keep the same format as the original field. 32 | 33 | Do not output any redundant explanation other than JSON. 34 | ''' 35 | 36 | 37 | async def translate( 38 | vid: str, 39 | cid: str, 40 | lang: str, 41 | openai_api_key: str = '', 42 | ) -> Optional[Translation]: 43 | chapter = find_chapter_by_cid(cid) 44 | if not chapter: 45 | abort(404, f'translate, but chapter not found, vid={vid}, cid={cid}') # nopep8. 46 | 47 | # Avoid the same language. 48 | la = Language.get(lang) 49 | lb = Language.get(chapter.lang) 50 | if la.language == lb.language: 51 | return None 52 | 53 | trans = find_translation(vid=vid, cid=cid, lang=lang) 54 | if trans and trans.chapter and trans.summary: 55 | return trans 56 | 57 | system_prompt = _TRANSLATION_SYSTEM_PROMPT.format(lang=lang) 58 | system_message = build_message(Role.SYSTEM, system_prompt) 59 | user_message = build_message(Role.SYSTEM, json.dumps({ 60 | 'chapter': chapter.chapter, 61 | 'summary': chapter.summary, 62 | }, ensure_ascii=False)) 63 | 64 | # Don't check token limit here, let it go. 65 | messages = [system_message, user_message] 66 | tokens = count_tokens(messages) 67 | logger.info(f'translate, vid={vid}, cid={cid}, lang={lang}, tokens={tokens}') # nopep8. 68 | 69 | body = await chat( 70 | messages=messages, 71 | model=Model.GPT_3_5_TURBO, 72 | top_p=0.1, 73 | timeout=90, 74 | api_key=openai_api_key, 75 | ) 76 | 77 | content = get_content(body) 78 | logger.info(f'translate, vid={vid}, cid={cid}, lang={lang}, content=\n{content}') # nopep8. 79 | 80 | # FIXME (Matthew Lee) prompt output as JSON may not work. 81 | res: dict = json.loads(content) 82 | chapter = res.get('chapter', '').strip() 83 | summary = res.get('summary', '').strip() 84 | 85 | # Both fields must exist. 86 | if (not chapter) or (not summary): 87 | abort(500, f'translate, but chapter or summary empty, vid={vid}, cid={cid}, lang={lang}') # nopep8. 88 | 89 | trans = Translation( 90 | vid=vid, 91 | cid=cid, 92 | lang=lang, 93 | chapter=chapter, 94 | summary=summary, 95 | ) 96 | 97 | insert_or_update_translation(trans) 98 | return trans 99 | --------------------------------------------------------------------------------