├── .gitignore ├── .idea ├── .gitignore ├── misc.xml ├── modules.xml ├── rusty_pipe.iml └── vcs.xml ├── Cargo.toml ├── LICENSE ├── examples ├── example_channel.rs ├── example_playlist.rs ├── example_stream.rs ├── example_trending.rs ├── examples_search.rs └── quickjs_test.rs └── src ├── downloader_trait.rs ├── lib.rs ├── utils ├── mod.rs └── utils.rs └── youtube_extractor ├── channel_extractor.rs ├── channel_info_item_extractor.rs ├── error.rs ├── itag_item.rs ├── mod.rs ├── playlist_extractor.rs ├── playlist_info_item_extractor.rs ├── search_extractor.rs ├── stream_extractor.rs ├── stream_info_item_extractor.rs └── trending_extractor.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | Cargo.lock 4 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /workspace.xml -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/rusty_pipe.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rusty_pipe" 3 | version = "0.1.0" 4 | authors = ["Deep Gaurav "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | regex = "1.3.3" 11 | serde_json = "1.0.45" 12 | pcre2 = "0.2.3" 13 | # fancy-regex = "0.3.5" 14 | lazy_static = "1.4.0" 15 | percent-encoding = "2.1.0" 16 | async-trait = "0.1.30" 17 | serde = { version = "1.0", features = ["derive"] } 18 | futures = "0.3.4" 19 | failure = "0.1.8" 20 | log = "0.4" 21 | 22 | [dev-dependencies] 23 | reqwest = { version = "0.10.4", features = ["json"] } 24 | urlencoding = "1.0.0" 25 | tokio = { version = "0.2", features = ["macros"] } 26 | quick-js = "0.3.0" 27 | pretty_env_logger = "0.3" 28 | Boa = "0.11.0" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /examples/example_channel.rs: -------------------------------------------------------------------------------- 1 | extern crate rusty_pipe; 2 | 3 | use rusty_pipe::youtube_extractor::search_extractor::*; 4 | use std::io; 5 | 6 | use rusty_pipe::downloader_trait::Downloader; 7 | use std::collections::HashMap; 8 | use std::str::FromStr; 9 | use urlencoding::encode; 10 | 11 | use async_trait::async_trait; 12 | use failure::Error; 13 | use rusty_pipe::youtube_extractor::channel_extractor::YTChannelExtractor; 14 | use rusty_pipe::youtube_extractor::error::ParsingError; 15 | use rusty_pipe::youtube_extractor::stream_info_item_extractor::YTStreamInfoItemExtractor; 16 | use serde_json::Value; 17 | 18 | struct DownloaderExample; 19 | 20 | #[async_trait] 21 | impl Downloader for DownloaderExample { 22 | async fn download(&self,url: &str) -> Result { 23 | println!("query url : {}", url); 24 | let resp = reqwest::get(url) 25 | .await 26 | .map_err(|er| ParsingError::DownloadError { 27 | cause: er.to_string(), 28 | })?; 29 | println!("got response "); 30 | let body = resp 31 | .text() 32 | .await 33 | .map_err(|er| ParsingError::DownloadError { 34 | cause: er.to_string(), 35 | })?; 36 | println!("suceess query"); 37 | Ok(String::from(body)) 38 | } 39 | 40 | async fn download_with_header(&self, 41 | url: &str, 42 | header: HashMap, 43 | ) -> Result { 44 | let client = reqwest::Client::new(); 45 | let res = client.get(url); 46 | let mut headers = reqwest::header::HeaderMap::new(); 47 | for header in header { 48 | headers.insert( 49 | reqwest::header::HeaderName::from_str(&header.0).map_err(|e| e.to_string())?, 50 | header.1.parse().unwrap(), 51 | ); 52 | } 53 | let res = res.headers(headers); 54 | let res = res.send().await.map_err(|er| er.to_string())?; 55 | let body = res.text().await.map_err(|er| er.to_string())?; 56 | Ok(String::from(body)) 57 | } 58 | 59 | async fn eval_js(&self,script: &str) -> Result { 60 | use quick_js::{Context, JsValue}; 61 | let context = Context::new().expect("Cant create js context"); 62 | // println!("decryption code \n{}",decryption_code); 63 | // println!("signature : {}",encrypted_sig); 64 | println!("jscode \n{}", script); 65 | let res = context.eval(script).unwrap_or(quick_js::JsValue::Null); 66 | // println!("js result : {:?}", result); 67 | let result = res.into_string().unwrap_or("".to_string()); 68 | println!("JS result: {}", result); 69 | Ok(result) 70 | } 71 | } 72 | 73 | fn print_videos(videos: Vec) { 74 | let mut count = 0; 75 | for vid in videos { 76 | count += 1; 77 | println!("STREAM {}", count); 78 | println!("title: {:#?}", vid.get_name()); 79 | } 80 | } 81 | 82 | #[tokio::main] 83 | async fn main() -> Result<(), Error> { 84 | println!("Enter channel id: "); 85 | let mut channel_id = String::new(); 86 | std::io::stdin() 87 | .read_line(&mut channel_id) 88 | .expect("Input failed"); 89 | channel_id = channel_id.trim().to_string(); 90 | let channel_extractor = YTChannelExtractor::new(DownloaderExample,&channel_id, None).await?; 91 | println!("Channel name {:#?}", channel_extractor.get_name()); 92 | println!( 93 | "Channel Thumbnails \n{:#?}", 94 | channel_extractor.get_avatars() 95 | ); 96 | println!("Channel Banners \n{:#?}", channel_extractor.get_banners()); 97 | println!("Videos :\n"); 98 | // print_videos(channel_extractor.get_videos()?); 99 | let mut videos = vec![]; 100 | videos.append(&mut channel_extractor.get_videos()?); 101 | println!( 102 | "Next Page url: {:#?}", 103 | channel_extractor.get_next_page_url() 104 | ); 105 | 106 | let mut next_page_url = channel_extractor.get_next_page_url()?; 107 | 108 | while let Some(next_page) = next_page_url.clone() { 109 | let extractor = 110 | YTChannelExtractor::new(DownloaderExample, &channel_id, Some(next_page)).await?; 111 | // print_videos(extractor.get_videos()?); 112 | next_page_url = extractor.get_next_page_url()?; 113 | videos.append(&mut channel_extractor.get_videos()?); 114 | println!("Next page url {:#?}", next_page_url); 115 | } 116 | print_videos(videos); 117 | 118 | Ok(()) 119 | } 120 | -------------------------------------------------------------------------------- /examples/example_playlist.rs: -------------------------------------------------------------------------------- 1 | extern crate rusty_pipe; 2 | 3 | use rusty_pipe::youtube_extractor::search_extractor::*; 4 | use std::io; 5 | 6 | use rusty_pipe::downloader_trait::Downloader; 7 | use std::collections::HashMap; 8 | use std::str::FromStr; 9 | use urlencoding::encode; 10 | 11 | use async_trait::async_trait; 12 | use failure::Error; 13 | use rusty_pipe::youtube_extractor::channel_extractor::YTChannelExtractor; 14 | use rusty_pipe::youtube_extractor::error::ParsingError; 15 | use rusty_pipe::youtube_extractor::playlist_extractor::YTPlaylistExtractor; 16 | use rusty_pipe::youtube_extractor::stream_info_item_extractor::YTStreamInfoItemExtractor; 17 | use serde_json::Value; 18 | 19 | struct DownloaderExample; 20 | 21 | #[async_trait] 22 | impl Downloader for DownloaderExample { 23 | async fn download(&self,url: &str) -> Result { 24 | println!("query url : {}", url); 25 | let resp = reqwest::get(url) 26 | .await 27 | .map_err(|er| ParsingError::DownloadError { 28 | cause: er.to_string(), 29 | })?; 30 | println!("got response "); 31 | let body = resp 32 | .text() 33 | .await 34 | .map_err(|er| ParsingError::DownloadError { 35 | cause: er.to_string(), 36 | })?; 37 | println!("suceess query"); 38 | Ok(String::from(body)) 39 | } 40 | 41 | async fn download_with_header(&self, 42 | url: &str, 43 | header: HashMap, 44 | ) -> Result { 45 | let client = reqwest::Client::new(); 46 | let res = client.get(url); 47 | let mut headers = reqwest::header::HeaderMap::new(); 48 | for header in header { 49 | headers.insert( 50 | reqwest::header::HeaderName::from_str(&header.0).map_err(|e| e.to_string())?, 51 | header.1.parse().unwrap(), 52 | ); 53 | } 54 | let res = res.headers(headers); 55 | let res = res.send().await.map_err(|er| er.to_string())?; 56 | let body = res.text().await.map_err(|er| er.to_string())?; 57 | Ok(String::from(body)) 58 | } 59 | 60 | async fn eval_js(&self,script: &str) -> Result { 61 | use quick_js::{Context, JsValue}; 62 | let context = Context::new().expect("Cant create js context"); 63 | // println!("decryption code \n{}",decryption_code); 64 | // println!("signature : {}",encrypted_sig); 65 | println!("jscode \n{}", script); 66 | let res = context.eval(script).unwrap_or(quick_js::JsValue::Null); 67 | // println!("js result : {:?}", result); 68 | let result = res.into_string().unwrap_or("".to_string()); 69 | print!("JS result: {}", result); 70 | Ok(result) 71 | } 72 | } 73 | 74 | fn print_videos(videos: Vec) { 75 | let mut count = 0; 76 | for vid in videos { 77 | count += 1; 78 | println!("STREAM {}", count); 79 | println!("title: {:#?}", vid.get_name()); 80 | } 81 | } 82 | 83 | #[tokio::main] 84 | async fn main() -> Result<(), Error> { 85 | println!("Enter playlist id: "); 86 | let mut playlist_id = String::new(); 87 | std::io::stdin() 88 | .read_line(&mut playlist_id) 89 | .expect("Input failed"); 90 | playlist_id = playlist_id.trim().to_string(); 91 | let playlist_extractor = 92 | YTPlaylistExtractor::new(&playlist_id, DownloaderExample, None).await?; 93 | println!("Playlist name {:#?}", playlist_extractor.get_name()); 94 | println!( 95 | "Playlist Thumbnails \n{:#?}", 96 | playlist_extractor.get_thumbnails() 97 | ); 98 | println!( 99 | "Uploader name: {:#?}", 100 | playlist_extractor.get_uploader_name() 101 | ); 102 | println!("Uploader url: {:#?}", playlist_extractor.get_uploader_url()); 103 | println!( 104 | "Uploaders thumbnails \n{:#?}", 105 | playlist_extractor.get_uploader_avatars() 106 | ); 107 | 108 | println!( 109 | "Videos count : {:#?}", 110 | playlist_extractor.get_stream_count() 111 | ); 112 | 113 | println!("Videos :\n"); 114 | // print_videos(channel_extractor.get_videos()?); 115 | let mut videos = vec![]; 116 | videos.append(&mut playlist_extractor.get_videos()?); 117 | println!( 118 | "Next Page url: {:#?}", 119 | playlist_extractor.get_next_page_url() 120 | ); 121 | 122 | let mut next_page_url = playlist_extractor.get_next_page_url()?; 123 | 124 | while let Some(next_page) = next_page_url.clone() { 125 | let extractor = 126 | YTPlaylistExtractor::new(&playlist_id, DownloaderExample, Some(next_page)).await?; 127 | // print_videos(extractor.get_videos()?); 128 | next_page_url = extractor.get_next_page_url()?; 129 | videos.append(&mut playlist_extractor.get_videos()?); 130 | println!("Next page url {:#?}", next_page_url); 131 | } 132 | print_videos(videos); 133 | 134 | Ok(()) 135 | } 136 | -------------------------------------------------------------------------------- /examples/example_stream.rs: -------------------------------------------------------------------------------- 1 | extern crate rusty_pipe; 2 | 3 | use rusty_pipe::downloader_trait::Downloader; 4 | use rusty_pipe::youtube_extractor::search_extractor::*; 5 | use rusty_pipe::youtube_extractor::stream_extractor::*; 6 | use std::io; 7 | 8 | use async_trait::async_trait; 9 | use rusty_pipe::youtube_extractor::error::ParsingError; 10 | use std::collections::hash_map::RandomState; 11 | use std::collections::HashMap; 12 | use std::str::FromStr; 13 | use urlencoding::encode; 14 | 15 | #[tokio::main] 16 | async fn main() -> Result<(), failure::Error> { 17 | pretty_env_logger::init(); 18 | static APP_USER_AGENT: &str = 19 | "Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/43.0"; 20 | 21 | // let url = "https://www.youtube.com/watch?v=09R8_2nJtjg&disable_polymer=1"; 22 | // let resp = reqwest::get(url).await.map_err(|er|er.to_string())?; 23 | 24 | let downloader = DownloaderExample {}; 25 | // let body = D::download(url).await?; 26 | 27 | let mut stream_extractor = YTStreamExtractor::new("7EvwIw4gIyk", downloader).await?; 28 | // let video_streams = stream_extractor.get_video_stream()?; 29 | // println!("AUDIO/VIDEO STREAMS \n"); 30 | // println!("{:#?}", video_streams); 31 | 32 | let audio_streams = stream_extractor.get_audio_streams().await?; 33 | println!("AUDIO ONLY STREAMS \n"); 34 | println!("{:#?}", audio_streams); 35 | 36 | // let video_only_streams = stream_extractor.get_video_only_stream()?; 37 | // println!("VIDEO ONLY STREAMS \n"); 38 | // println!("{:#?}", video_only_streams); 39 | 40 | let thumbnails = stream_extractor.get_video_thumbnails(); 41 | println!("\nTHUMBNAILS"); 42 | println!("{:#?}", thumbnails); 43 | 44 | println!("\nMETADATA"); 45 | println!("title: {:#?}", stream_extractor.get_name()); 46 | println!( 47 | "description:\n{:#?}", 48 | stream_extractor.get_description(false) 49 | ); 50 | println!("duration: {:#?}", stream_extractor.get_length()); 51 | println!("views: {:#?}", stream_extractor.get_view_count()); 52 | println!("likes: {:#?}", stream_extractor.get_like_count()); 53 | println!("dislikes: {:#?}", stream_extractor.get_dislike_count()); 54 | println!("uploader url: {:#?}", stream_extractor.get_uploader_url()); 55 | println!("uploader name: {:#?}", stream_extractor.get_uploader_name()); 56 | println!( 57 | "uploader thumbnails:\n {:#?}", 58 | stream_extractor.get_uploader_avatar_url() 59 | ); 60 | // println!("is live : {:#?}",stream_extractor.is_live()); 61 | Ok(()) 62 | } 63 | 64 | struct DownloaderExample; 65 | 66 | #[async_trait] 67 | impl Downloader for DownloaderExample { 68 | async fn download(&self,url: &str) -> Result { 69 | println!("query url : {}", url); 70 | let resp = reqwest::get(url) 71 | .await 72 | .map_err(|er| ParsingError::DownloadError { 73 | cause: er.to_string(), 74 | })?; 75 | println!("got response "); 76 | let body = resp 77 | .text() 78 | .await 79 | .map_err(|er| ParsingError::DownloadError { 80 | cause: er.to_string(), 81 | })?; 82 | println!("suceess query"); 83 | Ok(String::from(body)) 84 | } 85 | 86 | async fn download_with_header(&self, 87 | url: &str, 88 | header: HashMap, 89 | ) -> Result { 90 | let client = reqwest::Client::new(); 91 | let res = client.get(url); 92 | let mut headers = reqwest::header::HeaderMap::new(); 93 | for header in header { 94 | headers.insert( 95 | reqwest::header::HeaderName::from_str(&header.0).map_err(|e| e.to_string())?, 96 | header.1.parse().unwrap(), 97 | ); 98 | } 99 | let res = res.headers(headers); 100 | let res = res.send().await.map_err(|er| er.to_string())?; 101 | let body = res.text().await.map_err(|er| er.to_string())?; 102 | Ok(String::from(body)) 103 | } 104 | 105 | async fn eval_js(&self,script: &str) -> Result { 106 | use quick_js::{Context, JsValue}; 107 | let context = Context::new().expect("Cant create js context"); 108 | // println!("decryption code \n{}",decryption_code); 109 | // println!("signature : {}",encrypted_sig); 110 | println!("jscode \n{}", script); 111 | let res = context.eval(script).unwrap_or(quick_js::JsValue::Null); 112 | // println!("js result : {:?}", result); 113 | let result = res.into_string().unwrap_or("".to_string()); 114 | println!("JS result: {}", result); 115 | Ok(result) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /examples/example_trending.rs: -------------------------------------------------------------------------------- 1 | extern crate rusty_pipe; 2 | 3 | use async_trait::async_trait; 4 | use failure::Error; 5 | use rusty_pipe::downloader_trait::Downloader; 6 | use rusty_pipe::youtube_extractor::error::ParsingError; 7 | use rusty_pipe::youtube_extractor::stream_info_item_extractor::YTStreamInfoItemExtractor; 8 | use rusty_pipe::youtube_extractor::trending_extractor::YTTrendingExtractor; 9 | use std::collections::HashMap; 10 | use std::str::FromStr; 11 | 12 | struct DownloaderExample; 13 | 14 | #[async_trait] 15 | impl Downloader for DownloaderExample { 16 | async fn download(&self,url: &str) -> Result { 17 | println!("query url : {}", url); 18 | let resp = reqwest::get(url) 19 | .await 20 | .map_err(|er| ParsingError::DownloadError { 21 | cause: er.to_string(), 22 | })?; 23 | println!("got response "); 24 | let body = resp 25 | .text() 26 | .await 27 | .map_err(|er| ParsingError::DownloadError { 28 | cause: er.to_string(), 29 | })?; 30 | println!("suceess query"); 31 | Ok(String::from(body)) 32 | } 33 | 34 | async fn download_with_header(&self, 35 | url: &str, 36 | header: HashMap, 37 | ) -> Result { 38 | let client = reqwest::Client::new(); 39 | let res = client.get(url); 40 | let mut headers = reqwest::header::HeaderMap::new(); 41 | for header in header { 42 | headers.insert( 43 | reqwest::header::HeaderName::from_str(&header.0).map_err(|e| e.to_string())?, 44 | header.1.parse().unwrap(), 45 | ); 46 | } 47 | let res = res.headers(headers); 48 | let res = res.send().await.map_err(|er| er.to_string())?; 49 | let body = res.text().await.map_err(|er| er.to_string())?; 50 | Ok(String::from(body)) 51 | } 52 | 53 | async fn eval_js(&self,script: &str) -> Result { 54 | use quick_js::{Context, JsValue}; 55 | let context = Context::new().expect("Cant create js context"); 56 | // println!("decryption code \n{}",decryption_code); 57 | // println!("signature : {}",encrypted_sig); 58 | println!("jscode \n{}", script); 59 | let res = context.eval(script).unwrap_or(quick_js::JsValue::Null); 60 | // println!("js result : {:?}", result); 61 | let result = res.into_string().unwrap_or("".to_string()); 62 | print!("JS result: {}", result); 63 | Ok(result) 64 | } 65 | } 66 | 67 | fn print_videos(videos: Vec) { 68 | let mut count = 0; 69 | for vid in videos { 70 | count += 1; 71 | println!("STREAM {}", count); 72 | println!("title: {:#?}", vid.get_name()); 73 | } 74 | } 75 | 76 | #[tokio::main] 77 | async fn main() -> Result<(), Error> { 78 | let extractor = YTTrendingExtractor::new(DownloaderExample).await?; 79 | 80 | let videos = extractor.get_videos()?; 81 | 82 | print_videos(videos); 83 | Ok(()) 84 | } 85 | -------------------------------------------------------------------------------- /examples/examples_search.rs: -------------------------------------------------------------------------------- 1 | extern crate rusty_pipe; 2 | 3 | use rusty_pipe::youtube_extractor::search_extractor::*; 4 | use std::io; 5 | 6 | use rusty_pipe::downloader_trait::Downloader; 7 | use std::collections::HashMap; 8 | use std::str::FromStr; 9 | use urlencoding::encode; 10 | 11 | use async_trait::async_trait; 12 | use failure::Error; 13 | use rusty_pipe::youtube_extractor::error::ParsingError; 14 | 15 | #[tokio::main] 16 | async fn main() -> Result<(), Error> { 17 | let mut search_query = String::new(); 18 | println!("Enter Search Query"); 19 | io::stdin() 20 | .read_line(&mut search_query) 21 | .expect("Cannot Read Input"); 22 | 23 | search_query = encode(&search_query); 24 | 25 | let search_extractor = YTSearchExtractor::new(&search_query, None,DownloaderExample).await?; 26 | let search_suggestion = 27 | YTSearchExtractor::get_search_suggestion(&search_query,&DownloaderExample).await?; 28 | 29 | println!("Search suggestion {:#?}", search_suggestion); 30 | let mut items = search_extractor.search_results()?; 31 | let mut next_url = search_extractor.get_next_page_url()?; 32 | println!("Next page url : {:#?}", next_url); 33 | let mut max_page = 5; 34 | while let Some(url) = next_url.clone() { 35 | max_page -= 1; 36 | if max_page < 0 { 37 | break; 38 | } 39 | let search_extractor = 40 | YTSearchExtractor::new(&search_query, Some(url),DownloaderExample).await?; 41 | items.append(&mut search_extractor.search_results()?); 42 | next_url = search_extractor.get_next_page_url()?; 43 | println!("Next page url : {:#?}", next_url); 44 | } 45 | println!("Items Found {}", items.len()); 46 | println!(); 47 | 48 | for item in items { 49 | match item { 50 | YTSearchItem::StreamInfoItem(streaminfoitem) => { 51 | println!("Stream"); 52 | println!( 53 | "title : {}", 54 | streaminfoitem.get_name().expect("Stream has no title") 55 | ); 56 | println!("id: {:#?}", streaminfoitem.video_id()); 57 | println!( 58 | "URL : {}", 59 | streaminfoitem.get_url().expect("Stream has no url") 60 | ); 61 | println!("isLive: {:#?}", streaminfoitem.is_live()); 62 | println!("Duration: {:#?}", streaminfoitem.get_duration()); 63 | println!( 64 | "Uploader: {:#?}", 65 | streaminfoitem 66 | .get_uploader_name() 67 | .unwrap_or("Unknown".to_string()) 68 | ); 69 | println!( 70 | "Uploader Url: {}", 71 | streaminfoitem 72 | .get_uploader_url() 73 | .unwrap_or("Unknown".to_owned()) 74 | ); 75 | println!( 76 | "Upload Date: {:#?}", 77 | streaminfoitem.get_textual_upload_date() 78 | ); 79 | println!("View Count: {:#?}", streaminfoitem.get_view_count()); 80 | println!("Thumbnails:\n {:#?}", streaminfoitem.get_thumbnails()); 81 | println!( 82 | "Uploader Thumbnails:\n {:#?}", 83 | streaminfoitem.get_uploader_thumbnails() 84 | ); 85 | 86 | println!(); 87 | } 88 | YTSearchItem::ChannelInfoItem(channel_info_item) => { 89 | println!("Channel"); 90 | println!( 91 | "Name : {}", 92 | channel_info_item 93 | .get_name() 94 | .unwrap_or("Unknown".to_string()) 95 | ); 96 | println!("Channel Id : {:#?}", channel_info_item.channel_id()); 97 | println!( 98 | "Url : {}", 99 | channel_info_item.get_url().unwrap_or("Unknown".to_owned()) 100 | ); 101 | println!("Thumbnails \n{:#?}", channel_info_item.get_thumbnails()); 102 | println!( 103 | "Subscriber's count : {:#?}", 104 | channel_info_item.get_subscriber_count() 105 | ); 106 | println!("Description : {:#?}", channel_info_item.get_description()); 107 | println!( 108 | "Stream Count : {}", 109 | channel_info_item 110 | .get_stream_count() 111 | .map_or("Unknown".to_owned(), |c| c.to_string()) 112 | ); 113 | 114 | println!(); 115 | } 116 | YTSearchItem::PlaylistInfoItem(playlist_info_item) => { 117 | println!("Playlist"); 118 | println!( 119 | "Name : {}", 120 | playlist_info_item 121 | .get_name() 122 | .unwrap_or("Unknown".to_owned()) 123 | ); 124 | println!( 125 | "Url : {}", 126 | playlist_info_item.get_url().unwrap_or("Unknown".to_owned()) 127 | ); 128 | println!("Thumbnails \n{:#?}", playlist_info_item.get_thumbnails()); 129 | println!( 130 | "Uploader Name : {}", 131 | playlist_info_item 132 | .get_uploader_name() 133 | .unwrap_or("Unknown".to_string()) 134 | ); 135 | println!( 136 | "Stream Count : {:#?}", 137 | playlist_info_item.get_stream_count() 138 | ); 139 | 140 | println!(); 141 | } 142 | } 143 | } 144 | 145 | Ok(()) 146 | } 147 | 148 | struct DownloaderExample; 149 | 150 | #[async_trait] 151 | impl Downloader for DownloaderExample { 152 | async fn download(&self,url: &str) -> Result { 153 | println!("query url : {}", url); 154 | let resp = reqwest::get(url) 155 | .await 156 | .map_err(|er| ParsingError::DownloadError { 157 | cause: er.to_string(), 158 | })?; 159 | println!("got response "); 160 | let body = resp 161 | .text() 162 | .await 163 | .map_err(|er| ParsingError::DownloadError { 164 | cause: er.to_string(), 165 | })?; 166 | println!("suceess query"); 167 | Ok(String::from(body)) 168 | } 169 | 170 | async fn download_with_header(&self, 171 | url: &str, 172 | header: HashMap, 173 | ) -> Result { 174 | let client = reqwest::Client::new(); 175 | let res = client.get(url); 176 | let mut headers = reqwest::header::HeaderMap::new(); 177 | for header in header { 178 | headers.insert( 179 | reqwest::header::HeaderName::from_str(&header.0).map_err(|e| e.to_string())?, 180 | header.1.parse().unwrap(), 181 | ); 182 | } 183 | let res = res.headers(headers); 184 | let res = res.send().await.map_err(|er| er.to_string())?; 185 | let body = res.text().await.map_err(|er| er.to_string())?; 186 | Ok(String::from(body)) 187 | } 188 | 189 | async fn eval_js(&self,script: &str) -> Result { 190 | use quick_js::{Context, JsValue}; 191 | let context = Context::new().expect("Cant create js context"); 192 | // println!("decryption code \n{}",decryption_code); 193 | // println!("signature : {}",encrypted_sig); 194 | println!("jscode \n{}", script); 195 | let res = context.eval(script).unwrap_or(quick_js::JsValue::Null); 196 | // println!("js result : {:?}", result); 197 | let result = res.into_string().unwrap_or("".to_string()); 198 | print!("JS result: {}", result); 199 | Ok(result) 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /examples/quickjs_test.rs: -------------------------------------------------------------------------------- 1 | extern crate rusty_pipe; 2 | use quick_js::{Context, JsValue}; 3 | 4 | fn main() { 5 | let context = Context::new().unwrap(); 6 | let r = context.eval("function sum(a,b){return a+b;} sum(2,5)"); 7 | println!("{:?}", r); 8 | 9 | let r2 = context.eval(r##" 10 | var Fv={ 11 | Cc:function(a,b){a.splice(0,b)}, 12 | UF:function(a){a.reverse()}, 13 | Vw:function(a,b){ 14 | var c=a[0]; 15 | a[0]=a[b%a.length]; 16 | a[b%a.length]=c 17 | } 18 | }; 19 | var Gv=function(a){ 20 | a=a.split(""); 21 | Fv.Vw(a,25); 22 | Fv.Vw(a,47); 23 | Fv.UF(a,63); 24 | Fv.Cc(a,1); 25 | Fv.Vw(a,47); 26 | return a.join("") 27 | }; 28 | function decrypt(a){return Gv(a);};decrypt("M=AOzsJhJF_0tQ657YzWo2o_dI4GAFEjmi2l6f1uml7e3CQ=C0qfVIBJA13JjabpXp9nsd14cLOQa8i0nZY3ZlYoU2wQgIQRwsLlPpJCC") 29 | "##); 30 | 31 | println!("{:?}", r2); 32 | } 33 | -------------------------------------------------------------------------------- /src/downloader_trait.rs: -------------------------------------------------------------------------------- 1 | use crate::youtube_extractor::error::ParsingError; 2 | use async_trait::async_trait; 3 | use failure::Error; 4 | use std::collections::HashMap; 5 | 6 | #[cfg(target_arch = "wasm32")] 7 | #[async_trait(?Send)] 8 | pub trait Downloader { 9 | async fn download(&self,url: &str) -> Result; 10 | async fn download_with_header( 11 | &self, 12 | url: &str, 13 | header: HashMap, 14 | ) -> Result; 15 | async fn eval_js(&self,script: &str) -> Result; 16 | } 17 | 18 | #[cfg(not(target_arch = "wasm32"))] 19 | #[async_trait()] 20 | pub trait Downloader { 21 | async fn download(&self,url: &str) -> Result; 22 | async fn download_with_header( 23 | &self, 24 | url: &str, 25 | header: HashMap, 26 | ) -> Result; 27 | async fn eval_js( 28 | &self, 29 | script: &str) -> Result; 30 | } 31 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod downloader_trait; 2 | pub mod utils; 3 | pub mod youtube_extractor; 4 | 5 | #[cfg(test)] 6 | mod tests { 7 | #[test] 8 | fn it_works() { 9 | assert_eq!(2 + 2, 4); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod utils; 2 | -------------------------------------------------------------------------------- /src/utils/utils.rs: -------------------------------------------------------------------------------- 1 | use crate::youtube_extractor::error::ParsingError; 2 | use serde_json::Value; 3 | use std::collections::hash_map::HashMap; 4 | 5 | pub fn remove_non_digit_chars(input: &str) -> Result { 6 | let re = regex::Regex::new("\\D+").unwrap(); 7 | let onlydigits = re.replace_all(input, ""); 8 | let count = onlydigits.parse::(); 9 | count 10 | } 11 | 12 | pub fn mixed_number_word_parse(input: &str) -> Result { 13 | // println!("input string to parse: {}",input); 14 | let re = regex::Regex::new(r##"[\d]+([\.,][\d]+)?([KMBkmb])+"##).expect("regex incrorrect"); 15 | let mut multiplier = String::new(); 16 | if let Some(cap) = re.captures(input) { 17 | if let Some(grp) = cap.get(2) { 18 | multiplier = grp.as_str().to_string(); 19 | } 20 | } 21 | let mut count_str = String::new(); 22 | let re1 = regex::Regex::new(r##"([\d]+([\.,][\d]+)?)"##).expect("Regex incorrect"); 23 | if let Some(cap) = re1.captures(input) { 24 | if let Some(grp) = cap.get(0) { 25 | count_str = grp.as_str().replace(",", "."); 26 | } 27 | } 28 | let mut count = count_str 29 | .parse::() 30 | .map_err(|e| ParsingError::from(format!("{:#?},count_str = {}", e, count_str)))?; 31 | match multiplier.to_uppercase().as_str() { 32 | "K" => count = count * 1000_f32, 33 | "M" => count = count * 1000_f32 * 1000_f32, 34 | "B" => count = count * 1000_f32 * 1000_f32 * 1000_f32, 35 | _ => count = count, 36 | } 37 | Ok(count as i32) 38 | } 39 | 40 | pub fn compat_parse_map(input: &str) -> HashMap { 41 | let mut map = HashMap::new(); 42 | for arg in input.split("&") { 43 | let split_arg: Vec<&str> = arg.split("=").collect(); 44 | if let Some(arg_p) = split_arg.get(0) { 45 | map.insert( 46 | format!("{}", arg_p), 47 | String::from( 48 | percent_encoding::percent_decode_str(&format!( 49 | "{}", 50 | split_arg.get(1).unwrap_or(&"") 51 | )) 52 | .decode_utf8_lossy(), 53 | ), 54 | ); 55 | } 56 | } 57 | map 58 | } 59 | 60 | pub fn fix_thumbnail_url(url: &str) -> String { 61 | if url.starts_with("//") { 62 | format!("https:{}", url) 63 | } else { 64 | if url.starts_with("http") { 65 | url.to_string() 66 | } else { 67 | format!("https://{}", url) 68 | } 69 | } 70 | } 71 | 72 | pub fn get_url_from_navigation_endpoint( 73 | navigation_endpoint: &Value, 74 | ) -> Result { 75 | if let Some(intern_url) = navigation_endpoint 76 | .get("urlEndpoint") 77 | .and_then(|ue| ue.get("url")) 78 | .and_then(|ue| ue.as_str()) 79 | { 80 | if intern_url.starts_with("/redirect?") { 81 | let intern_url = &intern_url[10..]; 82 | for param in intern_url.split("&") { 83 | if let Some(first_param) = param.split("=").next() { 84 | if first_param == "q" { 85 | let url = { 86 | if let Some(urlencoded) = param.split("=").nth(1) { 87 | String::from( 88 | percent_encoding::percent_decode_str(urlencoded) 89 | .decode_utf8_lossy(), 90 | ) 91 | } else { 92 | String::new() 93 | } 94 | }; 95 | return Ok(url); 96 | } 97 | } 98 | } 99 | } else if intern_url.starts_with("http") { 100 | return Ok(intern_url.to_string()); 101 | } 102 | } else if let Some(browser_endpoint) = navigation_endpoint.get("browseEndpoint") { 103 | let canonical_base_url = browser_endpoint 104 | .get("canonicalBaseUrl") 105 | .and_then(|c| c.as_str()); 106 | let browse_id = browser_endpoint.get("browseId").and_then(|c| c.as_str()); 107 | 108 | if let Some(browse_id) = browse_id { 109 | if browse_id.starts_with("UC") { 110 | return Ok(format!("https://www.youtube.com/channel/{}", browse_id)); 111 | } 112 | } 113 | if let Some(base_url) = canonical_base_url { 114 | if !base_url.is_empty() { 115 | return Ok(format!("https://www.youtube.com{}", base_url)); 116 | } 117 | } 118 | return Err(ParsingError::parsing_error_from_str( 119 | "Canonical base url is none, browse id is not channel", 120 | )); 121 | } else if let Some(watch_endpoint) = navigation_endpoint.get("watchEndpoint") { 122 | let mut url = format!( 123 | "https://www.youtube.com/watch?v={}", 124 | watch_endpoint 125 | .get("videoId") 126 | .and_then(|f| f.as_str()) 127 | .unwrap_or("") 128 | ); 129 | if let Some(playlist_id) = watch_endpoint.get("playlistId").and_then(|f| f.as_str()) { 130 | url = url + "&list=" + playlist_id; 131 | } 132 | if let Some(start_time_sec) = watch_endpoint 133 | .get("startTimeSeconds") 134 | .and_then(|f| f.as_str()) 135 | { 136 | url = url + "&t=" + start_time_sec; 137 | } 138 | return Ok(url); 139 | } else if let Some(watch_playlist) = navigation_endpoint 140 | .get("watchPlaylistEndpoint") 141 | .and_then(|f| f.as_str()) 142 | { 143 | return Ok(format!( 144 | "https://www.youtube.com/playlist?list={}", 145 | watch_playlist 146 | )); 147 | } 148 | Ok("".to_string()) 149 | } 150 | 151 | pub fn get_text_from_object( 152 | text_object: &Value, 153 | html: bool, 154 | ) -> Result, ParsingError> { 155 | if let Some(simple_text) = text_object.get("simpleText") { 156 | return Ok(Some( 157 | simple_text 158 | .as_str() 159 | .ok_or("simple text not string")? 160 | .to_string(), 161 | )); 162 | } 163 | let mut text = String::new(); 164 | if let Some(runs) = text_object.get("runs").and_then(|runs| runs.as_array()) { 165 | for text_part in runs { 166 | let mut text_p = text_part 167 | .get("text") 168 | .and_then(|p| p.as_str()) 169 | .unwrap_or("") 170 | .to_string(); 171 | if html { 172 | if let Some(navp) = text_part.get("navigationEndpoint") { 173 | let url = get_url_from_navigation_endpoint(navp)?; 174 | if !url.is_empty() { 175 | text += &format!("{}", url, text_p); 176 | continue; 177 | } 178 | } 179 | } 180 | text += &text_p; 181 | } 182 | if html { 183 | text = text.replace("\n", "
"); 184 | text = text.replace(" ", "  "); 185 | } 186 | return Ok(Some(text)); 187 | } else { 188 | return Ok(None); 189 | } 190 | } 191 | 192 | pub fn match_to_closing_paranthesis(string: &str, start: &str) -> Option { 193 | // let string = string.to_string(); 194 | let start_index = string.find(start)?; 195 | let start_index = start_index + start.len(); 196 | let mut end_index = start_index; 197 | while string.chars().nth(end_index)? != '{' { 198 | end_index += 1; 199 | } 200 | end_index += 1; 201 | let mut open_paranthesis = 1; 202 | while open_paranthesis > 0 { 203 | match string.chars().nth(end_index)? { 204 | '{' => { 205 | open_paranthesis += 1; 206 | } 207 | '}' => { 208 | open_paranthesis -= 1; 209 | } 210 | _ => {} 211 | } 212 | end_index += 1; 213 | } 214 | Some(string.to_string()[start_index..end_index].to_string()) 215 | } 216 | -------------------------------------------------------------------------------- /src/youtube_extractor/channel_extractor.rs: -------------------------------------------------------------------------------- 1 | use crate::downloader_trait::Downloader; 2 | use crate::utils::utils::{fix_thumbnail_url, get_text_from_object}; 3 | use crate::youtube_extractor::error::ParsingError; 4 | use crate::youtube_extractor::stream_extractor::{Thumbnail, HARDCODED_CLIENT_VERSION}; 5 | use crate::youtube_extractor::stream_info_item_extractor::YTStreamInfoItemExtractor; 6 | use futures::try_join; 7 | use serde_json::{Map, Value}; 8 | use std::collections::HashMap; 9 | 10 | pub static CHANNEL_URL_BASE: &str = "https://www.youtube.com/channel/"; 11 | 12 | #[derive(Clone, PartialEq)] 13 | pub struct YTChannelExtractor{ 14 | initial_data: Value, 15 | video_tab: Value, 16 | downloader:D, 17 | page: Option<(Vec, Option)>, 18 | } 19 | 20 | impl YTChannelExtractor { 21 | async fn get_initial_data(id: &str,downloader:&D) -> Result { 22 | let mut url = format!("{}{}/videos?pbj=1&view=0&flow=grid", CHANNEL_URL_BASE, id); 23 | 24 | let mut level = 0; 25 | let mut ajax_json = Value::Null; 26 | while level < 3 { 27 | let mut headers = HashMap::new(); 28 | headers.insert("X-YouTube-Client-Name".to_string(), "1".to_string()); 29 | headers.insert( 30 | "X-YouTube-Client-Version".to_string(), 31 | HARDCODED_CLIENT_VERSION.to_string(), 32 | ); 33 | let response = downloader.download_with_header(&url, headers).await?; 34 | let json_response = serde_json::from_str::(&response) 35 | .map_err(|e| ParsingError::from(e.to_string()))?; 36 | let endpoint = (|| { 37 | json_response 38 | .get(1)? 39 | .get("response")? 40 | .get("onResponseReceivedActions")? 41 | .get(0)? 42 | .get("navigateAction")? 43 | .get("endpoint") 44 | })() 45 | .unwrap_or(&Value::Null); 46 | let webpage_type = (|| { 47 | endpoint 48 | .get("commandMetadata")? 49 | .get("webCommandMetadata")? 50 | .get("webPageType")? 51 | .as_str() 52 | })() 53 | .unwrap_or_default(); 54 | let browse_id = (|| endpoint.get("browseEndpoint")?.get("browseId")?.as_str())() 55 | .unwrap_or_default(); 56 | 57 | if webpage_type.eq_ignore_ascii_case("WEB_PAGE_TYPE_BROWSE") && !browse_id.is_empty() { 58 | if !browse_id.starts_with("UC") { 59 | return Err(ParsingError::from( 60 | "Redirected id is not pointing to a channel", 61 | )); 62 | } 63 | url = format!( 64 | "https://www.youtube.com/channel/{}/videos?pbj=1&view=0&flow=grid", 65 | browse_id 66 | ); 67 | level += 1; 68 | } else { 69 | ajax_json = json_response; 70 | break; 71 | } 72 | } 73 | 74 | if ajax_json == Value::Null { 75 | Err(ParsingError::from("Could not fetch initial JSON data")) 76 | } else { 77 | let init_data = 78 | (|| ajax_json.get(1)?.get("response"))().ok_or("reponse null in ajax json")?; 79 | Ok(init_data.clone()) 80 | } 81 | } 82 | 83 | fn get_video_tab(initial_data: &Value) -> Result { 84 | let tabs = (|| { 85 | initial_data 86 | .get("contents")? 87 | .get("twoColumnBrowseResultsRenderer")? 88 | .get("tabs")? 89 | .as_array() 90 | })() 91 | .ok_or("Tabs not found")?; 92 | let mut video_tab = &Value::Null; 93 | 94 | for tab in tabs { 95 | if let Some(renderer) = tab.get("tabRenderer") { 96 | if renderer 97 | .get("title") 98 | .unwrap_or(&Value::Null) 99 | .as_str() 100 | .unwrap_or_default() 101 | == "Videos" 102 | { 103 | video_tab = renderer; 104 | break; 105 | } 106 | } 107 | } 108 | 109 | if video_tab == &Value::Null { 110 | return Err(ParsingError::from("This channel has no Videos tab")); 111 | } 112 | let message_renderer_text = (|| { 113 | Some(get_text_from_object( 114 | video_tab 115 | .get("content")? 116 | .get("sectionListRenderer")? 117 | .get("contents")? 118 | .get(0)? 119 | .get("itemSectionRenderer")? 120 | .get("contents")? 121 | .get("0")? 122 | .get("messageRenderer")? 123 | .get("text")?, 124 | false, 125 | )) 126 | })(); 127 | 128 | if let Some(message) = message_renderer_text { 129 | if let Some(message) = message? { 130 | if message == "This channel has no videos." { 131 | return Ok(Value::Null); 132 | } 133 | } 134 | } 135 | Ok(video_tab.clone()) 136 | } 137 | 138 | pub async fn new( 139 | downloader:D, 140 | channel_id: &str, 141 | page_url: Option, 142 | ) -> Result { 143 | if let Some(page_url) = page_url { 144 | let initial_data = Self::get_initial_data(channel_id,&downloader); 145 | let page = Self::get_page(&page_url,&downloader); 146 | use futures::try_join; 147 | let (initial_data, page) = try_join!(initial_data, page)?; 148 | let video_tab = Self::get_video_tab(&initial_data)?; 149 | 150 | Ok(YTChannelExtractor { 151 | initial_data, 152 | video_tab, 153 | page: Some(page), 154 | downloader 155 | }) 156 | } else { 157 | let initial_data = Self::get_initial_data(channel_id,&downloader).await?; 158 | let video_tab = Self::get_video_tab(&initial_data)?; 159 | Ok(YTChannelExtractor { 160 | initial_data, 161 | video_tab, 162 | page: None, 163 | downloader 164 | }) 165 | } 166 | } 167 | 168 | fn collect_streams_from( 169 | videos: &Value, 170 | ) -> Result, ParsingError> { 171 | let mut streams = vec![]; 172 | for video in videos.as_array().ok_or("videos not array")? { 173 | if let Some(vid_renderer) = video.get("gridVideoRenderer") { 174 | if let Value::Object(video_info) = vid_renderer { 175 | streams.push(YTStreamInfoItemExtractor { 176 | video_info: video_info.clone(), 177 | }); 178 | } 179 | } 180 | } 181 | 182 | Ok(streams) 183 | } 184 | 185 | fn get_next_page_url_from(continuation: &Value) -> Option { 186 | let next_continuation_data = continuation.get(0)?.get("nextContinuationData")?; 187 | let continuation = next_continuation_data.get("continuation")?.as_str()?; 188 | let click_tracking_params = next_continuation_data 189 | .get("clickTrackingParams")? 190 | .as_str()?; 191 | Some(format!( 192 | "https://www.youtube.com/browse_ajax?ctoken={}&continuation={}&itct={}", 193 | continuation, continuation, click_tracking_params 194 | )) 195 | } 196 | 197 | async fn get_page( 198 | page_url: &str, 199 | downloader:&D, 200 | ) -> Result<(Vec, Option), ParsingError> { 201 | let mut headers = HashMap::new(); 202 | headers.insert("X-YouTube-Client-Name".to_string(), "1".to_string()); 203 | headers.insert( 204 | "X-YouTube-Client-Version".to_string(), 205 | HARDCODED_CLIENT_VERSION.to_string(), 206 | ); 207 | let response = downloader.download_with_header(&page_url, headers).await?; 208 | let json_response = serde_json::from_str::(&response) 209 | .map_err(|e| ParsingError::from(e.to_string()))?; 210 | 211 | let section_list_continuation = (|| { 212 | json_response 213 | .get(1)? 214 | .get("response")? 215 | .get("continuationContents")? 216 | .get("gridContinuation") 217 | })() 218 | .ok_or("Cant get continuation")?; 219 | 220 | let items = Self::collect_streams_from( 221 | section_list_continuation 222 | .get("items") 223 | .ok_or("items not in continuation")?, 224 | )?; 225 | let next_url = Self::get_next_page_url_from( 226 | section_list_continuation 227 | .get("continuations") 228 | .unwrap_or(&Value::Null), 229 | ); 230 | 231 | Ok((items, next_url)) 232 | } 233 | } 234 | 235 | impl YTChannelExtractor { 236 | pub fn get_name(&self) -> Result { 237 | Ok((|| { 238 | self.initial_data 239 | .get("header")? 240 | .get("c4TabbedHeaderRenderer")? 241 | .get("title")? 242 | .as_str() 243 | })() 244 | .ok_or("Cant get title")? 245 | .to_string()) 246 | } 247 | 248 | pub fn get_avatars(&self) -> Result, ParsingError> { 249 | let mut thumbnails = vec![]; 250 | for thumb in self 251 | .initial_data 252 | .get("header") 253 | .ok_or("No header")? 254 | .get("c4TabbedHeaderRenderer") 255 | .ok_or("No c4tabbed header")? 256 | .get("avatar") 257 | .ok_or("no avatar")? 258 | .get("thumbnails") 259 | .ok_or("no thumbnails")? 260 | .as_array() 261 | .ok_or("thumbnails array")? 262 | { 263 | // println!("{:#?}",thumb); 264 | if let Ok(thumb) = serde_json::from_value(thumb.to_owned()) { 265 | // thumb.url = fix_thumbnail_url(&thumb.url); 266 | thumbnails.push(thumb) 267 | } 268 | } 269 | Ok(thumbnails) 270 | } 271 | pub fn get_banners(&self) -> Result, ParsingError> { 272 | let mut thumbnails = vec![]; 273 | for thumb in self 274 | .initial_data 275 | .get("header") 276 | .ok_or("No header")? 277 | .get("c4TabbedHeaderRenderer") 278 | .ok_or("No c4tabbed header")? 279 | .get("banner") 280 | .ok_or("no banner")? 281 | .get("thumbnails") 282 | .ok_or("no thumbnails")? 283 | .as_array() 284 | .ok_or("thumbnails array")? 285 | { 286 | // println!("{:#?}",thumb); 287 | if let Ok(thumb) = serde_json::from_value(thumb.to_owned()) { 288 | // thumb.url = fix_thumbnail_url(&thumb.url); 289 | thumbnails.push(thumb) 290 | } 291 | } 292 | Ok(thumbnails) 293 | } 294 | 295 | pub fn get_videos(&self) -> Result, ParsingError> { 296 | if let Some((videos, _)) = &self.page { 297 | return Ok(videos.clone()); 298 | } 299 | let videos = (|| { 300 | self.video_tab 301 | .get("content")? 302 | .get("sectionListRenderer")? 303 | .get("contents")? 304 | .get(0)? 305 | .get("itemSectionRenderer")? 306 | .get("contents")? 307 | .get(0)? 308 | .get("gridRenderer")? 309 | .get("items") 310 | })() 311 | .ok_or("Cant get videos")?; 312 | Self::collect_streams_from(videos) 313 | } 314 | 315 | pub fn get_next_page_url(&self) -> Result, ParsingError> { 316 | if let Some((_, page_url)) = &self.page { 317 | Ok(page_url.clone()) 318 | } else { 319 | let conti = (|| { 320 | self.video_tab 321 | .get("content")? 322 | .get("sectionListRenderer")? 323 | .get("contents")? 324 | .get(0)? 325 | .get("itemSectionRenderer")? 326 | .get("contents")? 327 | .get(0)? 328 | .get("gridRenderer")? 329 | .get("continuations") 330 | })(); 331 | if let Some(conti) = conti { 332 | Ok(Self::get_next_page_url_from(conti)) 333 | } else { 334 | println!("Continuation is None"); 335 | Ok(None) 336 | } 337 | } 338 | } 339 | } 340 | -------------------------------------------------------------------------------- /src/youtube_extractor/channel_info_item_extractor.rs: -------------------------------------------------------------------------------- 1 | use crate::utils::utils::{get_text_from_object, mixed_number_word_parse, remove_non_digit_chars}; 2 | use crate::youtube_extractor::error::ParsingError; 3 | use crate::youtube_extractor::stream_extractor::Thumbnail; 4 | use serde_json::{Map, Value}; 5 | 6 | #[derive(Clone, PartialEq)] 7 | pub struct YTChannelInfoItemExtractor { 8 | pub channel_info: Map, 9 | } 10 | impl YTChannelInfoItemExtractor { 11 | pub fn get_thumbnails(&self) -> Result, ParsingError> { 12 | let mut thumbnails = vec![]; 13 | for thumb in self 14 | .channel_info 15 | .get("thumbnail") 16 | .ok_or("No thumbnail")? 17 | .get("thumbnails") 18 | .ok_or("no thumbnails")? 19 | .as_array() 20 | .ok_or("thumbnails array")? 21 | { 22 | if let Ok(thumb) = serde_json::from_value(thumb.to_owned()) { 23 | thumbnails.push(thumb) 24 | } 25 | } 26 | Ok(thumbnails) 27 | } 28 | 29 | pub fn get_name(&self) -> Result { 30 | if let Some(title) = self.channel_info.get("title") { 31 | let name = get_text_from_object(title, false)?; 32 | if let Some(name) = name { 33 | if !name.is_empty() { 34 | return Ok(name); 35 | } 36 | } 37 | } 38 | Err(ParsingError::from("Cannot get name")) 39 | } 40 | 41 | pub fn channel_id(&self) -> Result { 42 | let channel_id = self 43 | .channel_info 44 | .get("channelId") 45 | .ok_or("Cant get playlist id")? 46 | .as_str() 47 | .ok_or("Cant get playlist id")?; 48 | Ok(channel_id.to_string()) 49 | } 50 | 51 | pub fn get_url(&self) -> Result { 52 | Ok(format!( 53 | "https://www.youtube.com/channel/{}", 54 | self.channel_id()? 55 | )) 56 | } 57 | 58 | pub fn get_subscriber_count(&self) -> Result { 59 | if let Some(vct) = self.channel_info.get("subscriberCountText") { 60 | match get_text_from_object(vct, false) { 61 | Ok(uploader) => Ok(mixed_number_word_parse(&uploader.unwrap_or_default()) 62 | .map_err(|e| ParsingError::from(e.to_string()))?), 63 | Err(err) => Err(err), 64 | } 65 | } else { 66 | Ok(-1) 67 | } 68 | } 69 | 70 | pub fn get_stream_count(&self) -> Result { 71 | if let Some(vct) = self.channel_info.get("videoCountText") { 72 | match get_text_from_object(vct, false) { 73 | Ok(uploader) => Ok(remove_non_digit_chars::(&uploader.unwrap_or_default()) 74 | .map_err(|e| ParsingError::from(e.to_string()))?), 75 | Err(err) => Err(err), 76 | } 77 | } else { 78 | Ok(-1) 79 | } 80 | } 81 | 82 | pub fn get_description(&self) -> Result, ParsingError> { 83 | if let Some(vct) = self.channel_info.get("descriptionSnippet") { 84 | match get_text_from_object(vct, false) { 85 | Ok(description) => Ok(description), 86 | Err(err) => Err(err), 87 | } 88 | } else { 89 | Ok(None) 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/youtube_extractor/error.rs: -------------------------------------------------------------------------------- 1 | use failure::Error; 2 | use failure::Fail; 3 | use failure::_core::fmt::Debug; 4 | 5 | #[derive(Debug, Fail, Clone)] 6 | pub enum ParsingError { 7 | #[fail(display = "Parsing Error : {}", cause)] 8 | ParsingError { cause: String }, 9 | 10 | #[fail(display = "Age restricted video not supported")] 11 | AgeRestricted, 12 | 13 | #[fail(display = "Download Error : {}", cause)] 14 | DownloadError { cause: String }, 15 | } 16 | 17 | impl ParsingError { 18 | pub fn parsing_error_from_str(cause: &str) -> Self { 19 | ParsingError::ParsingError { 20 | cause: cause.to_string(), 21 | } 22 | } 23 | } 24 | 25 | impl From<&str> for ParsingError { 26 | fn from(cause: &str) -> Self { 27 | ParsingError::parsing_error_from_str(cause) 28 | } 29 | } 30 | 31 | impl From for ParsingError { 32 | fn from(cause: String) -> Self { 33 | ParsingError::ParsingError { cause } 34 | } 35 | } 36 | 37 | // impl From for ParsingError 38 | // where D:Debug 39 | // { 40 | // fn from(debug_msg: D) -> Self { 41 | // ParsingError::ParsingError { 42 | // cause:format!("{:#?}",debug_msg) 43 | // } 44 | // } 45 | // } 46 | -------------------------------------------------------------------------------- /src/youtube_extractor/itag_item.rs: -------------------------------------------------------------------------------- 1 | use lazy_static::lazy_static; 2 | 3 | use ItagType::*; 4 | 5 | lazy_static! { 6 | static ref ITAG_LIST: Vec = vec![ 7 | Itag { 8 | id: 17, 9 | itag_type: Video, 10 | resolution_string: String::from("240p"), 11 | ..Itag::default() 12 | }, 13 | Itag { 14 | id: 36, 15 | itag_type: Video, 16 | resolution_string: String::from("240p"), 17 | ..Itag::default() 18 | }, 19 | Itag { 20 | id: 18, 21 | itag_type: Video, 22 | resolution_string: String::from("360p"), 23 | ..Itag::default() 24 | }, 25 | Itag { 26 | id: 34, 27 | itag_type: Video, 28 | resolution_string: String::from("360p"), 29 | ..Itag::default() 30 | }, 31 | Itag { 32 | id: 35, 33 | itag_type: Video, 34 | resolution_string: String::from("480p"), 35 | ..Itag::default() 36 | }, 37 | Itag { 38 | id: 59, 39 | itag_type: Video, 40 | resolution_string: String::from("480p"), 41 | ..Itag::default() 42 | }, 43 | Itag { 44 | id: 78, 45 | itag_type: Video, 46 | resolution_string: String::from("480p"), 47 | ..Itag::default() 48 | }, 49 | Itag { 50 | id: 22, 51 | itag_type: Video, 52 | resolution_string: String::from("720p"), 53 | ..Itag::default() 54 | }, 55 | Itag { 56 | id: 37, 57 | itag_type: Video, 58 | resolution_string: String::from("1080p"), 59 | ..Itag::default() 60 | }, 61 | Itag { 62 | id: 38, 63 | itag_type: Video, 64 | resolution_string: String::from("1080p"), 65 | ..Itag::default() 66 | }, 67 | Itag { 68 | id: 43, 69 | itag_type: Video, 70 | resolution_string: String::from("360p"), 71 | ..Itag::default() 72 | }, 73 | Itag { 74 | id: 44, 75 | itag_type: Video, 76 | resolution_string: String::from("480p"), 77 | ..Itag::default() 78 | }, 79 | Itag { 80 | id: 45, 81 | itag_type: Video, 82 | resolution_string: String::from("720p"), 83 | ..Itag::default() 84 | }, 85 | Itag { 86 | id: 46, 87 | itag_type: Video, 88 | resolution_string: String::from("1080p"), 89 | ..Itag::default() 90 | }, 91 | Itag { 92 | id: 171, 93 | itag_type: Audio, 94 | avg_bitrate: 128, 95 | ..Itag::default() 96 | }, 97 | Itag { 98 | id: 172, 99 | itag_type: Audio, 100 | avg_bitrate: 256, 101 | ..Itag::default() 102 | }, 103 | Itag { 104 | id: 139, 105 | itag_type: Audio, 106 | avg_bitrate: 48, 107 | ..Itag::default() 108 | }, 109 | Itag { 110 | id: 140, 111 | itag_type: Audio, 112 | avg_bitrate: 128, 113 | ..Itag::default() 114 | }, 115 | Itag { 116 | id: 141, 117 | itag_type: Audio, 118 | avg_bitrate: 256, 119 | ..Itag::default() 120 | }, 121 | Itag { 122 | id: 249, 123 | itag_type: Audio, 124 | avg_bitrate: 50, 125 | ..Itag::default() 126 | }, 127 | Itag { 128 | id: 250, 129 | itag_type: Audio, 130 | avg_bitrate: 70, 131 | ..Itag::default() 132 | }, 133 | Itag { 134 | id: 251, 135 | itag_type: Audio, 136 | avg_bitrate: 160, 137 | ..Itag::default() 138 | }, 139 | Itag { 140 | id: 160, 141 | itag_type: VideoOnly, 142 | resolution_string: String::from("144p"), 143 | ..Itag::default() 144 | }, 145 | Itag { 146 | id: 133, 147 | itag_type: VideoOnly, 148 | resolution_string: String::from("240p"), 149 | ..Itag::default() 150 | }, 151 | Itag { 152 | id: 135, 153 | itag_type: VideoOnly, 154 | resolution_string: String::from("480p"), 155 | ..Itag::default() 156 | }, 157 | Itag { 158 | id: 212, 159 | itag_type: VideoOnly, 160 | resolution_string: String::from("480p"), 161 | ..Itag::default() 162 | }, 163 | Itag { 164 | id: 298, 165 | itag_type: VideoOnly, 166 | resolution_string: String::from("720p60"), 167 | fps: 60, 168 | ..Itag::default() 169 | }, 170 | Itag { 171 | id: 137, 172 | itag_type: VideoOnly, 173 | resolution_string: String::from("1080p"), 174 | ..Itag::default() 175 | }, 176 | Itag { 177 | id: 299, 178 | itag_type: VideoOnly, 179 | resolution_string: String::from("1080p60"), 180 | fps: 60, 181 | ..Itag::default() 182 | }, 183 | Itag { 184 | id: 266, 185 | itag_type: VideoOnly, 186 | resolution_string: String::from("2160p"), 187 | ..Itag::default() 188 | }, 189 | Itag { 190 | id: 278, 191 | itag_type: VideoOnly, 192 | resolution_string: String::from("144p"), 193 | ..Itag::default() 194 | }, 195 | Itag { 196 | id: 242, 197 | itag_type: VideoOnly, 198 | resolution_string: String::from("240p"), 199 | ..Itag::default() 200 | }, 201 | Itag { 202 | id: 244, 203 | itag_type: VideoOnly, 204 | resolution_string: String::from("480p"), 205 | ..Itag::default() 206 | }, 207 | Itag { 208 | id: 245, 209 | itag_type: VideoOnly, 210 | resolution_string: String::from("480p"), 211 | ..Itag::default() 212 | }, 213 | Itag { 214 | id: 246, 215 | itag_type: VideoOnly, 216 | resolution_string: String::from("480p"), 217 | ..Itag::default() 218 | }, 219 | Itag { 220 | id: 247, 221 | itag_type: VideoOnly, 222 | resolution_string: String::from("720p"), 223 | ..Itag::default() 224 | }, 225 | Itag { 226 | id: 248, 227 | itag_type: VideoOnly, 228 | resolution_string: String::from("1080p"), 229 | ..Itag::default() 230 | }, 231 | Itag { 232 | id: 271, 233 | itag_type: VideoOnly, 234 | resolution_string: String::from("1440p"), 235 | ..Itag::default() 236 | }, 237 | Itag { 238 | id: 272, 239 | itag_type: VideoOnly, 240 | resolution_string: String::from("2160p"), 241 | ..Itag::default() 242 | }, 243 | Itag { 244 | id: 302, 245 | itag_type: VideoOnly, 246 | resolution_string: String::from("720p60"), 247 | fps: 60, 248 | ..Itag::default() 249 | }, 250 | Itag { 251 | id: 303, 252 | itag_type: VideoOnly, 253 | resolution_string: String::from("1080p60"), 254 | fps: 60, 255 | ..Itag::default() 256 | }, 257 | Itag { 258 | id: 308, 259 | itag_type: VideoOnly, 260 | resolution_string: String::from("1440p60"), 261 | fps: 60, 262 | ..Itag::default() 263 | }, 264 | Itag { 265 | id: 313, 266 | itag_type: VideoOnly, 267 | resolution_string: String::from("2160p"), 268 | ..Itag::default() 269 | }, 270 | Itag { 271 | id: 315, 272 | itag_type: VideoOnly, 273 | resolution_string: String::from("2160p60"), 274 | fps: 60, 275 | ..Itag::default() 276 | }, 277 | Itag { 278 | id: 394, 279 | fps: 24, 280 | resolution_string: String::from("144p AV1"), 281 | itag_type: VideoOnly, 282 | ..Itag::default() 283 | }, 284 | Itag { 285 | id: 395, 286 | resolution_string: String::from("240p AV1"), 287 | itag_type: VideoOnly, 288 | ..Itag::default() 289 | }, 290 | Itag { 291 | id: 396, 292 | resolution_string: String::from("360p AV1"), 293 | itag_type: VideoOnly, 294 | ..Itag::default() 295 | }, 296 | Itag { 297 | id: 397, 298 | resolution_string: String::from("480p AV1"), 299 | itag_type: VideoOnly, 300 | ..Itag::default() 301 | }, 302 | Itag { 303 | id: 398, 304 | resolution_string: String::from("720p AV1"), 305 | itag_type: VideoOnly, 306 | ..Itag::default() 307 | }, 308 | Itag { 309 | id: 399, 310 | resolution_string: String::from("1080p AV1"), 311 | itag_type: VideoOnly, 312 | ..Itag::default() 313 | }, 314 | ]; 315 | } 316 | 317 | #[derive(PartialEq, Clone, Debug)] 318 | pub enum ItagType { 319 | Audio, 320 | Video, 321 | VideoOnly, 322 | } 323 | 324 | impl Default for ItagType { 325 | fn default() -> Self { 326 | Video 327 | } 328 | } 329 | 330 | #[derive(Default, Clone, Debug)] 331 | pub struct Itag { 332 | pub id: i64, 333 | pub itag_type: ItagType, 334 | pub avg_bitrate: i64, 335 | pub resolution_string: String, 336 | pub fps: i64, 337 | } 338 | 339 | impl Itag { 340 | pub fn is_supported(id: i64) -> bool { 341 | for itag in ITAG_LIST.iter() { 342 | if itag.id == id { 343 | return true; 344 | } 345 | } 346 | false 347 | } 348 | 349 | pub fn get_itag(id: i64) -> Result { 350 | for itag in ITAG_LIST.iter() { 351 | if itag.id == id { 352 | return Ok(itag.clone()); 353 | } 354 | } 355 | Err(format!("Itag id {} not found", id)) 356 | } 357 | } 358 | -------------------------------------------------------------------------------- /src/youtube_extractor/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod channel_extractor; 2 | pub mod channel_info_item_extractor; 3 | pub mod error; 4 | pub mod itag_item; 5 | pub mod playlist_extractor; 6 | pub mod playlist_info_item_extractor; 7 | pub mod search_extractor; 8 | pub mod stream_extractor; 9 | pub mod stream_info_item_extractor; 10 | pub mod trending_extractor; 11 | 12 | static YOUTUBE_BASE_URL: &str = "https://www.youtube.com"; 13 | 14 | fn fix_url(url: &str) -> String { 15 | if url.starts_with("/") { 16 | YOUTUBE_BASE_URL.to_owned() + url 17 | } else { 18 | url.to_owned() 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/youtube_extractor/playlist_extractor.rs: -------------------------------------------------------------------------------- 1 | use crate::downloader_trait::Downloader; 2 | use crate::utils::utils::{ 3 | get_text_from_object, get_url_from_navigation_endpoint, remove_non_digit_chars, 4 | }; 5 | use crate::youtube_extractor::error::ParsingError; 6 | use crate::youtube_extractor::stream_extractor::{Thumbnail, HARDCODED_CLIENT_VERSION}; 7 | use crate::youtube_extractor::stream_info_item_extractor::YTStreamInfoItemExtractor; 8 | use serde_json::Value; 9 | use std::collections::HashMap; 10 | 11 | #[derive(Clone, PartialEq)] 12 | pub struct YTPlaylistExtractor { 13 | init_data: Value, 14 | playlist_info: Value, 15 | page: Option<(Vec, Option)>, 16 | } 17 | 18 | impl YTPlaylistExtractor { 19 | pub async fn new( 20 | playlist_id: &str, 21 | downloader: D, 22 | page_url: Option, 23 | ) -> Result { 24 | if let Some(page_url) = page_url { 25 | let initial_data = YTPlaylistExtractor::get_initial_data(playlist_id, &downloader); 26 | let page = YTPlaylistExtractor::get_page(&page_url, &downloader); 27 | use futures::try_join; 28 | let (initial_data, page) = try_join!(initial_data, page)?; 29 | let playlist_info = YTPlaylistExtractor::get_playlist_info(&initial_data)?; 30 | 31 | Ok(Self { 32 | init_data: initial_data, 33 | playlist_info, 34 | page: Some(page), 35 | }) 36 | } else { 37 | let initial_data = 38 | YTPlaylistExtractor::get_initial_data(playlist_id, &downloader).await?; 39 | let playlist_info = YTPlaylistExtractor::get_playlist_info(&initial_data)?; 40 | Ok(Self { 41 | init_data: initial_data, 42 | playlist_info, 43 | page: None, 44 | }) 45 | } 46 | } 47 | 48 | async fn get_initial_data( 49 | id: &str, 50 | downloader: &D, 51 | ) -> Result { 52 | let url = format!("https://www.youtube.com/playlist?list={}&pbj=1", id); 53 | let mut headers = HashMap::new(); 54 | headers.insert("X-YouTube-Client-Name".to_string(), "1".to_string()); 55 | headers.insert( 56 | "X-YouTube-Client-Version".to_string(), 57 | HARDCODED_CLIENT_VERSION.to_string(), 58 | ); 59 | let response = downloader.download_with_header(&url, headers).await?; 60 | let json_response = serde_json::from_str::(&response) 61 | .map_err(|e| ParsingError::from(e.to_string()))?; 62 | let json_response = json_response 63 | .get(1) 64 | .ok_or("1 not in json resp")? 65 | .get("response") 66 | .ok_or("response not found")?; 67 | Ok(json_response.clone()) 68 | } 69 | 70 | fn get_playlist_info(initial_data: &Value) -> Result { 71 | let pinfo = (|| { 72 | initial_data 73 | .get("sidebar")? 74 | .get("playlistSidebarRenderer")? 75 | .get("items")? 76 | .get(0)? 77 | .get("playlistSidebarPrimaryInfoRenderer") 78 | })(); 79 | if let Some(pinfo) = pinfo { 80 | Ok(pinfo.clone()) 81 | } else { 82 | Err(ParsingError::from("Cant get playlist info")) 83 | } 84 | } 85 | 86 | fn collect_streams_from( 87 | videos: &Value, 88 | ) -> Result, ParsingError> { 89 | let mut streams = vec![]; 90 | let videos = videos.as_array().ok_or("Videos not array")?; 91 | for vid in videos { 92 | if let Some(video) = vid.get("playlistVideoRenderer") { 93 | if let Some(video) = video.as_object() { 94 | streams.push(YTStreamInfoItemExtractor { 95 | video_info: video.clone(), 96 | }) 97 | } 98 | } 99 | } 100 | Ok(streams) 101 | } 102 | 103 | fn get_next_page_url_from(continuation: &Value) -> Option { 104 | let next_continuation_data = continuation.get(0)?.get("nextContinuationData")?; 105 | let continuation = next_continuation_data.get("continuation")?.as_str()?; 106 | let click_tracking_params = next_continuation_data 107 | .get("clickTrackingParams")? 108 | .as_str()?; 109 | Some(format!( 110 | "https://www.youtube.com/browse_ajax?ctoken={}&continuation={}&itct={}", 111 | continuation, continuation, click_tracking_params 112 | )) 113 | } 114 | 115 | async fn get_page( 116 | page_url: &str, 117 | downloader: &D, 118 | ) -> Result<(Vec, Option), ParsingError> { 119 | let mut headers = HashMap::new(); 120 | headers.insert("X-YouTube-Client-Name".to_string(), "1".to_string()); 121 | headers.insert( 122 | "X-YouTube-Client-Version".to_string(), 123 | HARDCODED_CLIENT_VERSION.to_string(), 124 | ); 125 | let response = downloader.download_with_header(&page_url, headers).await?; 126 | let json_response = serde_json::from_str::(&response) 127 | .map_err(|e| ParsingError::from(e.to_string()))?; 128 | 129 | let section_list_continuation = (|| { 130 | json_response 131 | .get(1)? 132 | .get("response")? 133 | .get("continuationContents")? 134 | .get("playlistVideoListContinuation") 135 | })() 136 | .ok_or("Cant get continuation")?; 137 | 138 | let items = YTPlaylistExtractor::collect_streams_from( 139 | section_list_continuation 140 | .get("contents") 141 | .ok_or("items not in continuation")?, 142 | )?; 143 | let next_url = YTPlaylistExtractor::get_next_page_url_from( 144 | section_list_continuation 145 | .get("continuations") 146 | .unwrap_or(&Value::Null), 147 | ); 148 | 149 | Ok((items, next_url)) 150 | } 151 | } 152 | 153 | impl YTPlaylistExtractor { 154 | pub fn get_name(&self) -> Result { 155 | if let Some(title) = self.playlist_info.get("title") { 156 | let name = get_text_from_object(title, false)?; 157 | if let Some(name) = name { 158 | if !name.is_empty() { 159 | return Ok(name); 160 | } 161 | } 162 | } 163 | let title = (|| { 164 | self.init_data 165 | .get("microformat")? 166 | .get("microformatDataRenderer")? 167 | .get("title")? 168 | .as_str() 169 | })(); 170 | if let Some(title) = title { 171 | return Ok(title.to_string()); 172 | } 173 | Err(ParsingError::from("Cant get name")) 174 | } 175 | 176 | pub fn get_thumbnails(&self) -> Result, ParsingError> { 177 | let mut thumbnails = vec![]; 178 | for thumb in (|| { 179 | self.playlist_info 180 | .get("thumbnailRenderer")? 181 | .get("playlistVideoThumbnailRenderer")? 182 | .get("thumbnail")? 183 | .get("thumbnails")? 184 | .as_array() 185 | })() 186 | .or((|| { 187 | self.init_data 188 | .get("microformat")? 189 | .get("microformatDataRenderer")? 190 | .get("thumbnail")? 191 | .get("thumbnails")? 192 | .as_array() 193 | })()) 194 | .ok_or("Cant get thumbnails")? 195 | { 196 | // println!("{:#?}",thumb); 197 | if let Ok(thumb) = serde_json::from_value(thumb.to_owned()) { 198 | // thumb.url = fix_thumbnail_url(&thumb.url); 199 | thumbnails.push(thumb) 200 | } 201 | } 202 | Ok(thumbnails) 203 | } 204 | 205 | fn get_uploader_info(&self) -> Result { 206 | let items = (|| { 207 | self.init_data 208 | .get("sidebar")? 209 | .get("playlistSidebarRenderer")? 210 | .get("items")? 211 | .as_array() 212 | })(); 213 | if let Some(items) = items { 214 | for item in items { 215 | if let Some(video_owner) = (|| { 216 | item.get("playlistSidebarSecondaryInfoRenderer")? 217 | .get("videoOwner")? 218 | .get("videoOwnerRenderer") 219 | })() { 220 | return Ok(video_owner.clone()); 221 | } 222 | } 223 | } 224 | Err(ParsingError::from("Cant get uploader info")) 225 | } 226 | 227 | pub fn get_uploader_url(&self) -> Result { 228 | if let Some(navp) = self.get_uploader_info()?.get("navigationEndpoint") { 229 | return Ok(get_url_from_navigation_endpoint(navp)?); 230 | } else { 231 | Err(ParsingError::from("Cant get uploader url")) 232 | } 233 | } 234 | pub fn get_uploader_name(&self) -> Result { 235 | if let Some(navp) = self.get_uploader_info()?.get("title") { 236 | return Ok(get_text_from_object(navp, false)?.ok_or("uploader name not found")?); 237 | } else { 238 | Err(ParsingError::from("Cant get uploader url")) 239 | } 240 | } 241 | 242 | pub fn get_uploader_avatars(&self) -> Result, ParsingError> { 243 | let mut thumbnails = vec![]; 244 | let uploader = self.get_uploader_info()?; 245 | for thumb in (|| uploader.get("thumbnail")?.get("thumbnails")?.as_array())() 246 | .ok_or("Cant get uploaader thumbnails")? 247 | { 248 | // println!("{:#?}",thumb); 249 | if let Ok(thumb) = serde_json::from_value(thumb.to_owned()) { 250 | // thumb.url = fix_thumbnail_url(&thumb.url); 251 | thumbnails.push(thumb) 252 | } 253 | } 254 | Ok(thumbnails) 255 | } 256 | 257 | pub fn get_stream_count(&self) -> Result { 258 | let views_text = get_text_from_object( 259 | self.playlist_info 260 | .get("stats") 261 | .ok_or("No stats")? 262 | .get(0) 263 | .ok_or("No 0 in stats")?, 264 | false, 265 | )? 266 | .unwrap_or_default(); 267 | let videoc = remove_non_digit_chars::(&views_text) 268 | .map_err(|e| ParsingError::from(e.to_string()))?; 269 | Ok(videoc) 270 | } 271 | 272 | pub fn get_videos(&self) -> Result, ParsingError> { 273 | if let Some((videos, _)) = &self.page { 274 | return Ok(videos.clone()); 275 | } 276 | let videos = (|| { 277 | self.init_data 278 | .get("contents")? 279 | .get("twoColumnBrowseResultsRenderer")? 280 | .get("tabs")? 281 | .get(0)? 282 | .get("tabRenderer")? 283 | .get("content")? 284 | .get("sectionListRenderer")? 285 | .get("contents")? 286 | .get(0)? 287 | .get("itemSectionRenderer")? 288 | .get("contents")? 289 | .get(0)? 290 | .get("playlistVideoListRenderer")? 291 | .get("contents") 292 | })() 293 | .ok_or("Cant get videos")?; 294 | YTPlaylistExtractor::collect_streams_from(videos) 295 | } 296 | 297 | pub fn get_next_page_url(&self) -> Result, ParsingError> { 298 | if let Some((_, page_url)) = &self.page { 299 | Ok(page_url.clone()) 300 | } else { 301 | let conti = (|| { 302 | self.init_data 303 | .get("contents")? 304 | .get("twoColumnBrowseResultsRenderer")? 305 | .get("tabs")? 306 | .get(0)? 307 | .get("tabRenderer")? 308 | .get("content")? 309 | .get("sectionListRenderer")? 310 | .get("contents")? 311 | .get(0)? 312 | .get("itemSectionRenderer")? 313 | .get("contents")? 314 | .get(0)? 315 | .get("playlistVideoListRenderer")? 316 | .get("continuations") 317 | })(); 318 | if let Some(conti) = conti { 319 | Ok(YTPlaylistExtractor::get_next_page_url_from(conti)) 320 | } else { 321 | println!("Continuation is None"); 322 | Ok(None) 323 | } 324 | } 325 | } 326 | } 327 | -------------------------------------------------------------------------------- /src/youtube_extractor/playlist_info_item_extractor.rs: -------------------------------------------------------------------------------- 1 | use crate::utils::utils::remove_non_digit_chars; 2 | use crate::utils::utils::*; 3 | use crate::youtube_extractor::error::ParsingError; 4 | use crate::youtube_extractor::stream_extractor::Thumbnail; 5 | use serde_json::{Map, Value}; 6 | 7 | #[derive(Clone, PartialEq)] 8 | pub struct YTPlaylistInfoItemExtractor { 9 | pub playlist_info: Map, 10 | } 11 | 12 | impl YTPlaylistInfoItemExtractor { 13 | pub fn get_thumbnails(&self) -> Result, ParsingError> { 14 | let mut thumbnails = vec![]; 15 | for thumb in self 16 | .playlist_info 17 | .get("thumbnails") 18 | .ok_or("no thumbnails")? 19 | .as_array() 20 | .ok_or("thumbnails array")? 21 | { 22 | for thumb in thumb 23 | .get("thumbnails") 24 | .ok_or("no nested thumbnails")? 25 | .as_array() 26 | .ok_or("thumbnails array")? 27 | { 28 | if let Ok(thumb) = serde_json::from_value(thumb.to_owned()) { 29 | thumbnails.push(thumb) 30 | } 31 | } 32 | } 33 | Ok(thumbnails) 34 | } 35 | 36 | pub fn get_name(&self) -> Result { 37 | if let Some(title) = self.playlist_info.get("title") { 38 | let name = get_text_from_object(title, false)?; 39 | if let Some(name) = name { 40 | if !name.is_empty() { 41 | return Ok(name); 42 | } 43 | } 44 | } 45 | Err(ParsingError::from("Cannot get name")) 46 | } 47 | 48 | pub fn playlist_id(&self) -> Result { 49 | let playlist_id = self 50 | .playlist_info 51 | .get("playlistId") 52 | .ok_or("Cant get playlist id")? 53 | .as_str() 54 | .ok_or("Cant get playlist id")?; 55 | Ok(playlist_id.to_string()) 56 | } 57 | 58 | pub fn get_url(&self) -> Result { 59 | Ok(format!( 60 | "https://www.youtube.com/playlist?list={}", 61 | self.playlist_id()? 62 | )) 63 | } 64 | 65 | pub fn get_uploader_name(&self) -> Result { 66 | match get_text_from_object( 67 | self.playlist_info 68 | .get("longBylineText") 69 | .unwrap_or(&Value::Null), 70 | false, 71 | ) { 72 | Ok(uploader) => Ok(uploader.unwrap_or_default()), 73 | Err(err) => Err(err), 74 | } 75 | } 76 | 77 | pub fn get_stream_count(&self) -> Result { 78 | let vid_tex = self 79 | .playlist_info 80 | .get("videoCount") 81 | .unwrap_or(&Value::Null) 82 | .as_str() 83 | .unwrap_or_default(); 84 | Ok( 85 | remove_non_digit_chars::(vid_tex) 86 | .map_err(|e| ParsingError::from(e.to_string()))?, 87 | ) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/youtube_extractor/search_extractor.rs: -------------------------------------------------------------------------------- 1 | use super::super::downloader_trait::Downloader; 2 | use crate::youtube_extractor::channel_info_item_extractor::YTChannelInfoItemExtractor; 3 | use crate::youtube_extractor::error::ParsingError; 4 | use crate::youtube_extractor::playlist_info_item_extractor::YTPlaylistInfoItemExtractor; 5 | use crate::youtube_extractor::stream_extractor::HARDCODED_CLIENT_VERSION; 6 | use crate::youtube_extractor::stream_info_item_extractor::YTStreamInfoItemExtractor; 7 | use failure::Error; 8 | use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; 9 | use serde_json::{Map, Value}; 10 | use std::collections::HashMap; 11 | 12 | /// https://url.spec.whatwg.org/#fragment-percent-encode-set 13 | const FRAGMENT: &AsciiSet = &CONTROLS.add(b' ').add(b'"').add(b'<').add(b'>').add(b'`'); 14 | 15 | #[derive(Clone, PartialEq)] 16 | pub enum YTSearchItem { 17 | StreamInfoItem(YTStreamInfoItemExtractor), 18 | ChannelInfoItem(YTChannelInfoItemExtractor), 19 | PlaylistInfoItem(YTPlaylistInfoItemExtractor), 20 | } 21 | 22 | #[derive(Clone, PartialEq)] 23 | pub struct YTSearchExtractor { 24 | initial_data: Map, 25 | query: String, 26 | page: Option<(Vec, Option)>, 27 | p_url: Option, 28 | downloader:D, 29 | } 30 | 31 | impl YTSearchExtractor { 32 | async fn get_initial_data( 33 | url: &str, 34 | page_count: &str, 35 | downloader: &D 36 | ) -> Result, ParsingError> { 37 | let url = format!("{}&gl=US&pbj=1&page={}", url, page_count); 38 | let mut headers = HashMap::new(); 39 | headers.insert("X-YouTube-Client-Name".to_string(), "1".to_string()); 40 | headers.insert( 41 | "X-YouTube-Client-Version".to_string(), 42 | HARDCODED_CLIENT_VERSION.to_string(), 43 | ); 44 | let resp = downloader.download_with_header(&url, headers).await?; 45 | let resp_json = serde_json::from_str::(&resp) 46 | .map_err(|er| ParsingError::parsing_error_from_str(&er.to_string()))?; 47 | let resp_json = resp_json 48 | .get(1) 49 | .ok_or("index 1 not in pbj")? 50 | .get("response") 51 | .ok_or("response not in pbj")? 52 | .as_object() 53 | .ok_or(format!("initial data not json object "))? 54 | .to_owned(); 55 | Ok(resp_json) 56 | } 57 | 58 | pub fn collect_streams_from(videos: &Vec) -> Result, ParsingError> { 59 | let mut search_items = vec![]; 60 | for item in videos { 61 | if item.get("backgroundPromoRenderer").is_some() { 62 | return Err(ParsingError::from("Nothing found")); 63 | } 64 | if let Some(el) = item 65 | .get("videoRenderer") 66 | .or(item.get("compactVideoRenderer")) 67 | .map(|f| f.as_object()) 68 | { 69 | if let Some(vid_info) = el { 70 | search_items.push(YTSearchItem::StreamInfoItem(YTStreamInfoItemExtractor { 71 | video_info: vid_info.to_owned(), 72 | })); 73 | } 74 | } else if let Some(el) = item.get("channelRenderer").map(|f| f.as_object()) { 75 | if let Some(vid_info) = el { 76 | search_items.push(YTSearchItem::ChannelInfoItem(YTChannelInfoItemExtractor { 77 | channel_info: vid_info.to_owned(), 78 | })); 79 | } 80 | } else if let Some(el) = item.get("playlistRenderer").map(|f| f.as_object()) { 81 | if let Some(vid_info) = el { 82 | search_items.push(YTSearchItem::PlaylistInfoItem( 83 | YTPlaylistInfoItemExtractor { 84 | playlist_info: vid_info.to_owned(), 85 | }, 86 | )); 87 | } 88 | } 89 | } 90 | 91 | Ok(search_items) 92 | } 93 | 94 | fn get_next_page_url_from(continuation: &Value, query: &str) -> Option { 95 | // print!("{:#?}",continuation); 96 | let next_continuation_data = (|| continuation.get(0)?.get("nextContinuationData"))() 97 | .or((|| { 98 | continuation 99 | .get("continuationItemRenderer")? 100 | .get("continuationEndpoint") 101 | })()) 102 | .unwrap_or(&Value::Null); 103 | let continuation = next_continuation_data.get("continuation")?.as_str()?; 104 | let click_tracking_params = next_continuation_data 105 | .get("clickTrackingParams")? 106 | .as_str()?; 107 | Some(format!( 108 | "https://www.youtube.com/results?pbj=1&search_query={}&ctoken={}&continuation={}&itct={}", 109 | query,continuation, continuation, click_tracking_params 110 | )) 111 | } 112 | 113 | async fn get_page( 114 | page_url: &str, 115 | downloader: &D, 116 | query: &str, 117 | ) -> Result<(Vec, Option), ParsingError> { 118 | let mut headers = HashMap::new(); 119 | headers.insert("X-YouTube-Client-Name".to_string(), "1".to_string()); 120 | headers.insert( 121 | "X-YouTube-Client-Version".to_string(), 122 | HARDCODED_CLIENT_VERSION.to_string(), 123 | ); 124 | let response = downloader.download_with_header(&page_url, headers).await?; 125 | let json_response = serde_json::from_str::(&response) 126 | .map_err(|e| ParsingError::from(format!("json eror : {:#?}", e)))?; 127 | 128 | let section_list_continuation = (|| { 129 | json_response 130 | .get(1)? 131 | .get("response")? 132 | .get("continuationContents")? 133 | .get("itemSectionContinuation") 134 | })() 135 | .ok_or("Cant get continuation")?; 136 | 137 | let items = Self::collect_streams_from( 138 | section_list_continuation 139 | .get("contents") 140 | .ok_or("Not contents")? 141 | .as_array() 142 | .ok_or("items not in continuation")?, 143 | )?; 144 | let next_url = Self::get_next_page_url_from( 145 | section_list_continuation 146 | .get("continuations") 147 | .unwrap_or(&Value::Null), 148 | query, 149 | ); 150 | 151 | Ok((items, next_url)) 152 | } 153 | } 154 | 155 | impl YTSearchExtractor { 156 | pub async fn new( 157 | query: &str, 158 | page_url: Option, 159 | downloader:D, 160 | ) -> Result { 161 | let url = format!( 162 | "https://www.youtube.com/results?disable_polymer=1&search_query={}", 163 | query 164 | ); 165 | let query = utf8_percent_encode(query, FRAGMENT).to_string(); 166 | if let Some(page_url) = page_url { 167 | let initial_data = Self::get_initial_data(&url, &page_url,&downloader).await?; 168 | 169 | Ok(Self { 170 | initial_data, 171 | query, 172 | page: None, 173 | p_url: Some(page_url), 174 | downloader 175 | }) 176 | } else { 177 | let initial_data = Self::get_initial_data(&url, "1",&downloader).await?; 178 | Ok(Self { 179 | initial_data, 180 | query, 181 | page: None, 182 | p_url: Some("1".to_string()), 183 | downloader 184 | }) 185 | } 186 | } 187 | 188 | pub async fn get_search_suggestion( 189 | query: &str, 190 | downloader:&D, 191 | ) -> Result, ParsingError> { 192 | let mut suggestions = vec![]; 193 | let url = format!( 194 | "https://suggestqueries.google.com/complete/search\ 195 | ?client=youtube\ 196 | &jsonp=jp\ 197 | &ds=yt\ 198 | &q={}", 199 | query 200 | ); 201 | let resp = downloader.download(&url).await?; 202 | let resp = resp[3..resp.len() - 1].to_string(); 203 | let json = 204 | serde_json::from_str::(&resp).map_err(|e| ParsingError::from(e.to_string()))?; 205 | if let Some(collection) = (|| json.get(1)?.as_array())() { 206 | for suggestion in collection { 207 | if let Some(suggestion_str) = (|| suggestion.get(0)?.as_str())() { 208 | suggestions.push(suggestion_str.to_string()) 209 | } 210 | } 211 | } 212 | 213 | Ok(suggestions) 214 | } 215 | 216 | pub fn search_results(&self) -> Result, ParsingError> { 217 | if let Some((items, _)) = &self.page { 218 | return Ok(items.clone()); 219 | } 220 | // println!("{:#?}",self.initial_data); 221 | let sections = (|| { 222 | let data = self 223 | .initial_data 224 | .get("contents")? 225 | .get("twoColumnSearchResultsRenderer")? 226 | .get("primaryContents")? 227 | .get("sectionListRenderer")? 228 | .get("contents")? 229 | .as_array()?; 230 | Some(data) 231 | })() 232 | .ok_or("cant get sections ")?; 233 | 234 | let mut search_items: Vec = vec![]; 235 | 236 | for sect in sections { 237 | let item_section = (|| { 238 | let c = sect 239 | .get("itemSectionRenderer")? 240 | .get("contents")? 241 | .as_array()?; 242 | Some(c) 243 | })() 244 | .ok_or("cant get section"); 245 | if let Ok(item_section) = item_section { 246 | search_items.append(&mut Self::collect_streams_from(&item_section)?) 247 | } 248 | } 249 | return Ok(search_items); 250 | } 251 | 252 | pub fn get_next_page_url(&self) -> Result, ParsingError> { 253 | let pu = self 254 | .p_url 255 | .clone() 256 | .unwrap_or_default() 257 | .parse::() 258 | .map_err(|e| ParsingError::from(e.to_string()))?; 259 | return Ok(Some(format!("{}", pu + 1))); 260 | } 261 | } 262 | -------------------------------------------------------------------------------- /src/youtube_extractor/stream_extractor.rs: -------------------------------------------------------------------------------- 1 | use super::super::downloader_trait::Downloader; 2 | use serde::{Deserialize, Serialize}; 3 | use serde_json::{Map, Value}; 4 | use std::collections::HashMap; 5 | 6 | use super::super::utils::utils::*; 7 | use super::itag_item::{Itag, ItagType}; 8 | use crate::utils::utils; 9 | use crate::youtube_extractor::error::ParsingError; 10 | use crate::youtube_extractor::search_extractor::YTSearchItem; 11 | use failure::Error; 12 | use lazy_static::lazy_static; 13 | use std::future::Future; 14 | 15 | const CONTENT: &str = "content"; 16 | 17 | const FORMATS: &str = "formats"; 18 | const ADAPTIVE_FORMATS: &str = "adaptiveFormats"; 19 | const HTTPS: &str = "https:"; 20 | const DECRYPTION_FUNC_NAME: &str = "decrypt"; 21 | 22 | const VERIFIED_URL_PARAMS: &str = "&has_verified=1&bpctr=9999999999"; 23 | 24 | lazy_static! { 25 | static ref REGEXES: Vec<&'static str>=vec![ 26 | "(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)", 27 | "\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)", 28 | "\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)", 29 | "([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;", 30 | "\\b([\\w$]{2,})\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;", 31 | "\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(" 32 | ]; 33 | 34 | static ref N_PARAM_FUNC_REGEX:&'static str = "b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\S+)\\(b\\),a\\.set\\(\"n\",b\\)"; 35 | 36 | static ref N_PARAM_REGEX:&'static str ="[&?]n=([^&]+)"; 37 | } 38 | 39 | pub const HARDCODED_CLIENT_VERSION: &str = "2.20200214.04.00"; 40 | 41 | #[derive(Clone, PartialEq)] 42 | pub struct YTStreamExtractor { 43 | doc: String, 44 | // player_args: Map, 45 | // video_info_page:Map, 46 | player_response: Map, 47 | player_decryption_code: String, 48 | player_n_param_decryption_code: String, 49 | video_id: String, 50 | 51 | initial_data: Value, 52 | primary_info_renderer: Value, 53 | secondary_info_renderer: Value, 54 | // is_age_restricted:bool, 55 | downloader: D, 56 | } 57 | 58 | #[derive(Debug, Serialize, Deserialize)] 59 | pub struct StreamItem { 60 | pub url: Option, 61 | pub itag: u32, 62 | pub approxDurationMs: Option, 63 | pub audioChannels: Option, 64 | pub audioQuality: Option, 65 | pub audioSampleRate: Option, 66 | pub averageBitrate: Option, 67 | pub bitrate: u32, 68 | pub contentLength: Option, 69 | pub height: Option, 70 | pub width: Option, 71 | pub quality: String, 72 | pub qualityLabel: Option, 73 | pub lastModified: String, 74 | pub mimeType: String, 75 | } 76 | 77 | #[derive(Debug, Serialize, Deserialize)] 78 | pub struct Thumbnail { 79 | pub url: String, 80 | pub width: u64, 81 | pub height: u64, 82 | } 83 | 84 | impl YTStreamExtractor { 85 | pub async fn new(video_id: &str, downloader: D) -> Result { 86 | use futures::try_join; 87 | let url = format!( 88 | "https://www.youtube.com/watch?v={}&disable_polymer=1", 89 | video_id 90 | ); 91 | 92 | let doc = downloader.download(&url); 93 | let inital_ajax_json = Self::get_initial_ajax_json(&url, &downloader).await?; 94 | let initial_data = YTStreamExtractor::::get_initial_data(&inital_ajax_json); 95 | let (doc, initial_data) = try_join!(doc, initial_data)?; 96 | 97 | let initial_response = Self::get_player_response_from_initial_ajax(&inital_ajax_json); 98 | 99 | if initial_data.1 { 100 | return Err(ParsingError::AgeRestricted); 101 | } 102 | 103 | let initial_data = initial_data.0; 104 | let primary_info_renderer = 105 | YTStreamExtractor::::get_primary_info_renderer(&initial_data)?; 106 | let secondary_info_renderer = 107 | YTStreamExtractor::::get_secondary_info_renderer(&initial_data)?; 108 | if let Some(response) = initial_response { 109 | if Self::is_decryption_needed(&response).unwrap_or(true) { 110 | let player_url = Self::get_player_js_url(video_id, &downloader).await?; 111 | let player_code = 112 | YTStreamExtractor::::get_player_code(&player_url, &downloader).await?; 113 | let player_decryption_code = Self::load_decryption_code(&player_code)?; 114 | let player_n_param_decryption_code = 115 | Self::load_n_param_decryption_code(&player_code)?; 116 | Ok(YTStreamExtractor { 117 | player_response: response, 118 | downloader, 119 | player_decryption_code, 120 | initial_data, 121 | primary_info_renderer, 122 | secondary_info_renderer, 123 | doc: String::from(doc), 124 | video_id: String::from(video_id), 125 | player_n_param_decryption_code, 126 | }) 127 | } else { 128 | println!("Not loading nparam decryption"); 129 | Ok(YTStreamExtractor { 130 | player_response: response, 131 | downloader, 132 | player_decryption_code: "".to_owned(), 133 | initial_data, 134 | primary_info_renderer, 135 | secondary_info_renderer, 136 | doc: String::from(doc), 137 | video_id: String::from(video_id), 138 | player_n_param_decryption_code: "".to_owned(), 139 | }) 140 | } 141 | } else { 142 | // OLD METHOD 143 | let player_config = YTStreamExtractor::::get_player_config(&doc) 144 | .ok_or("cannot get player_config".to_string())?; 145 | // println!("player config : {:?}",player_config); 146 | 147 | let player_args = YTStreamExtractor::::get_player_args(&player_config) 148 | .ok_or("cannot get player args".to_string())?; 149 | // println!("player args : {:?} ",player_args); 150 | 151 | let player_response = YTStreamExtractor::::get_player_response(&player_args) 152 | .ok_or("cannot get player response".to_string())?; 153 | // println!("player response {:?}", player_response); 154 | let player_url = YTStreamExtractor::::get_player_url(&player_config) 155 | .ok_or("Cant get player url".to_owned())?; 156 | let player_code = 157 | YTStreamExtractor::::get_player_code(&player_url, &downloader).await?; 158 | let player_decryption_code = Self::load_decryption_code(&player_code)?; 159 | let player_n_param_decryption_code = Self::load_n_param_decryption_code(&player_code)?; 160 | Ok(YTStreamExtractor { 161 | player_response, 162 | downloader, 163 | player_decryption_code, 164 | player_n_param_decryption_code, 165 | initial_data, 166 | primary_info_renderer, 167 | secondary_info_renderer, 168 | doc: String::from(doc), 169 | video_id: String::from(video_id), 170 | }) 171 | } 172 | } 173 | 174 | fn is_decryption_needed(player_response: &Map) -> Result { 175 | let streaming_data = player_response.get("streamingData").unwrap_or(&Value::Null); 176 | if let Value::Object(streaming_data) = streaming_data { 177 | if let Value::Array(formats) = streaming_data.get(FORMATS).unwrap_or(&Value::Null) { 178 | if let Some(format_data) = formats.first() { 179 | match format_data.get("url").unwrap_or(&Value::Null) { 180 | Value::String(url) => Ok(false), 181 | _ => Ok(true), 182 | } 183 | } else { 184 | Ok(false) 185 | } 186 | 187 | // println!("all formats {:#?}",formats); 188 | } else { 189 | Ok(false) 190 | } 191 | } else { 192 | Err(ParsingError::ParsingError { 193 | cause: "Streaming data not found in player response".to_string(), 194 | }) 195 | } 196 | } 197 | 198 | async fn get_itags( 199 | streaming_data_key: &str, 200 | itag_type_wanted: ItagType, 201 | player_response: &Map, 202 | decryption_code: &str, 203 | n_param_decryption_code: &str, 204 | downloader: &D 205 | ) -> Result, ParsingError> { 206 | let mut url_and_itags = HashMap::new(); 207 | let streaming_data = player_response.get("streamingData").unwrap_or(&Value::Null); 208 | if let Value::Object(streaming_data) = streaming_data { 209 | if let Value::Array(formats) = streaming_data 210 | .get(streaming_data_key) 211 | .unwrap_or(&Value::Null) 212 | { 213 | // println!("all formats {:#?}",formats); 214 | for format_data in formats { 215 | if let Value::Object(format_data_obj) = format_data { 216 | // println!("format data {:#?}",format_data); 217 | 218 | let stream_url = match format_data_obj.get("url").unwrap_or(&Value::Null) { 219 | Value::String(url) => String::from(url), 220 | _ => { 221 | let cipherstr = { 222 | if let Value::String(cip) = format_data_obj 223 | .get("cipher") 224 | .or(format_data_obj.get("signatureCipher")) 225 | .unwrap_or(&Value::Null) 226 | { 227 | cip.clone() 228 | } else { 229 | String::default() 230 | } 231 | }; 232 | let cipher = compat_parse_map(&cipherstr); 233 | format!( 234 | "{}&{}={}", 235 | cipher.get("url").unwrap_or(&String::default()), 236 | cipher.get("sp").unwrap_or(&String::default()), 237 | &YTStreamExtractor::::decrypt_signature( 238 | cipher.get("s").unwrap_or(&"".to_owned()), 239 | decryption_code, 240 | downloader 241 | ).await 242 | ) 243 | } 244 | }; 245 | let stream_url = 246 | Self::apply_nparam_decryption(&stream_url, n_param_decryption_code,downloader).await; 247 | match serde_json::from_value::(format_data.clone()) { 248 | Ok(stream_item) => match itag_type_wanted { 249 | ItagType::VideoOnly => { 250 | if stream_item.audioQuality.is_none() { 251 | url_and_itags.insert(stream_url, stream_item); 252 | } 253 | } 254 | ItagType::Audio => { 255 | if stream_item.height.is_none() { 256 | url_and_itags.insert(stream_url, stream_item); 257 | } 258 | } 259 | _ => { 260 | url_and_itags.insert(stream_url, stream_item); 261 | } 262 | }, 263 | Err(err) => { 264 | // return Err(ParsingError::ParsingError { 265 | // cause: err.to_string(), 266 | // }) 267 | } 268 | } 269 | // url_and_itags.insert(stream_url, itag_item); 270 | } else { 271 | // println!("itag {} rejected",itag); 272 | } 273 | } 274 | } else { 275 | return Ok(url_and_itags); 276 | } 277 | } else { 278 | return Err(ParsingError::ParsingError { 279 | cause: "Streaming data not found in player response".to_string(), 280 | }); 281 | } 282 | 283 | Ok(url_and_itags) 284 | } 285 | 286 | pub fn parse_n_param(url: &str) -> Option { 287 | Self::match_group1(&N_PARAM_REGEX, url).ok() 288 | } 289 | 290 | pub async fn decrypt_n_param(n_param: &str, decryption_code: &str,downloader: &D) -> String { 291 | println!("Decrypt n param n_param {:#?} decryption code {:#?}",n_param,decryption_code); 292 | Self::decrypt_signature(n_param, decryption_code,downloader).await 293 | } 294 | 295 | pub fn replace_n_param(url: &str, old_param: &str, new_param: &str) -> String { 296 | log::info!("Replace param {} -> {}", old_param, new_param); 297 | url.replace(old_param, new_param) 298 | } 299 | 300 | pub async fn apply_nparam_decryption(url: &str, decryption_code: &str,downloader: &D) -> String { 301 | if let Some(old_param) = Self::parse_n_param(url) { 302 | let new_param = Self::decrypt_n_param(&old_param, decryption_code,downloader).await; 303 | Self::replace_n_param(url, &old_param, &new_param) 304 | } else { 305 | url.to_string() 306 | } 307 | } 308 | 309 | // pub fn get_video_streams() 310 | 311 | pub async fn get_player_code(player_url: &str, downloader: &D) -> Result { 312 | let player_url = { 313 | if player_url.starts_with("http://") || player_url.starts_with("https://") { 314 | player_url.to_string() 315 | } else { 316 | format!("https://youtube.com{}", player_url) 317 | } 318 | }; 319 | let player_code = downloader.download(&player_url).await?; 320 | Ok(player_code) 321 | } 322 | 323 | async fn decrypt_signature(encrypted_sig: &str, decryption_code: &str,downloader: &D) -> String { 324 | // println!("encrypted_sig: {:#?}", encrypted_sig); 325 | // println!("decryption_code {:#?}", decryption_code); 326 | 327 | let script = format!("{};decrypt(\"{}\")", decryption_code, encrypted_sig); 328 | let res = downloader.eval_js(&script).await; 329 | 330 | let result = res.unwrap_or_default(); 331 | 332 | result 333 | } 334 | 335 | fn get_player_config(page_html: &str) -> Option> { 336 | let pattern = regex::Regex::new(r"ytplayer.config\s*=\s*(\{.*?\});").ok()?; 337 | let grp = pattern.captures(page_html)?; 338 | let yt_player_config_raw = grp.get(1)?.as_str(); 339 | let v: Value = serde_json::from_str(yt_player_config_raw).ok()?; 340 | if let Value::Object(val) = v { 341 | return Some(val); 342 | } 343 | None 344 | } 345 | 346 | fn get_player_args(player_config: &Map) -> Option> { 347 | let args = player_config.get("args")?; 348 | if let Value::Object(args) = args { 349 | return Some(args.to_owned()); 350 | } 351 | None 352 | } 353 | 354 | fn fix_player_url(url: &str) -> String { 355 | let mut player_url = url.to_string(); 356 | if player_url.starts_with("//") { 357 | player_url = HTTPS.to_owned() + &player_url; 358 | } else if player_url.starts_with("/") { 359 | player_url = HTTPS.to_owned() + "//www.youtube.com" + &player_url; 360 | } 361 | player_url.to_string() 362 | } 363 | 364 | fn get_player_url(player_config: &Map) -> Option { 365 | let yt_assets = player_config.get("assets")?.as_object()?; 366 | let mut player_url = yt_assets.get("js")?.as_str()?.to_owned(); 367 | player_url = Self::fix_player_url(&player_url); 368 | Some(player_url) 369 | } 370 | 371 | fn get_player_response_from_initial_ajax( 372 | inital_ajax_json: &Value, 373 | ) -> Option> { 374 | let resp = inital_ajax_json.get(2)?.get("playerResponse")?; 375 | if let None = resp.get("streamingData") { 376 | None 377 | } else { 378 | Some(resp.as_object()?.clone()) 379 | } 380 | } 381 | 382 | fn get_player_response(player_args: &Map) -> Option> { 383 | let player_response_str = player_args.get("player_response")?.as_str()?; 384 | let player_response: Value = serde_json::from_str(player_response_str).ok()?; 385 | Some(player_response.as_object()?.to_owned()) 386 | } 387 | 388 | async fn get_initial_ajax_json(url: &str, downloader: &D) -> Result { 389 | let mut headers = HashMap::new(); 390 | headers.insert("X-YouTube-Client-Name".to_string(), "1".to_string()); 391 | headers.insert( 392 | "X-YouTube-Client-Version".to_string(), 393 | HARDCODED_CLIENT_VERSION.to_string(), 394 | ); 395 | let url = format!("{}&pbj=1", url); 396 | let data = downloader.download_with_header(&url, headers).await?; 397 | let initial_ajax_json: Value = serde_json::from_str(&data).map_err(|e| e.to_string())?; 398 | Ok(initial_ajax_json) 399 | } 400 | 401 | async fn get_player_js_url_iframeapi(downloader: &D) -> Result { 402 | let iframurl = "https://www.youtube.com/iframe_api"; 403 | let body = downloader.download(iframurl).await?; 404 | let hashPattern = "player\\\\\\/([a-z0-9]{8})\\\\\\/"; 405 | let hash = YTStreamExtractor::::match_group1(hashPattern, &body)?; 406 | Ok(format!( 407 | "https://www.youtube.com/s/player/{}/player_ias.vflset/en_US/base.js", 408 | hash 409 | )) 410 | } 411 | 412 | async fn get_player_js_url(video_id: &str, downloader: &D) -> Result { 413 | let player_url_c = Self::get_player_js_url_iframeapi(downloader).await; 414 | if let Ok(url) = player_url_c { 415 | return Ok(url); 416 | } 417 | let embed_url = format!("https://www.youtube.com/embed/{}", video_id); 418 | let mut headers = HashMap::new(); 419 | headers.insert("X-YouTube-Client-Name".to_string(), "1".to_string()); 420 | headers.insert( 421 | "X-YouTube-Client-Version".to_string(), 422 | HARDCODED_CLIENT_VERSION.to_string(), 423 | ); 424 | let data = downloader.download_with_header(&embed_url, headers).await?; 425 | let asset_pattern = "\"assets\":.+?\"js\":\\s*(\"[^\"]+\")"; 426 | let url1 = Self::match_group1(asset_pattern, &data); 427 | match url1 { 428 | Ok(url) => { 429 | let url = url.replace("\\", "").replace("\"", ""); 430 | let url = Self::fix_player_url(&url); 431 | Ok(url) 432 | } 433 | Err(_) => { 434 | let srcreg = r###"script.*src\s*="(.*)"\s*.*name\s*=\s*"player_ias\/base""###; 435 | let url = Self::match_group1(srcreg, &data); 436 | let url = url.and_then(|url| Ok(Self::fix_player_url(&url))); 437 | log::debug!("Player url {:#?}", url); 438 | url 439 | } 440 | } 441 | } 442 | 443 | async fn get_initial_data(initial_ajax_json: &Value) -> Result<(Value, bool), ParsingError> { 444 | let initial_ajax_json = initial_ajax_json 445 | .as_array() 446 | .ok_or("inital ajax json not array")?; 447 | if let Some(initial_data) = initial_ajax_json 448 | .get(2) 449 | .ok_or("inital ajax 2 not found")? 450 | .as_object() 451 | { 452 | if let Some(response) = initial_data.get("response") { 453 | Ok((response.clone(), true)) 454 | } else { 455 | if let Some(initial_data) = initial_ajax_json 456 | .get(3) 457 | .ok_or("initial ajax 2 not found")? 458 | .as_object() 459 | { 460 | if let Some(response) = initial_data.get("response") { 461 | Ok((response.clone(), false)) 462 | } else { 463 | Err(ParsingError::ParsingError { 464 | cause: "Cannot get initial data".to_string(), 465 | }) 466 | } 467 | } else { 468 | Err(ParsingError::ParsingError { 469 | cause: "initial ajax doesnt have index 3".to_string(), 470 | }) 471 | } 472 | } 473 | } else { 474 | Err(ParsingError::ParsingError { 475 | cause: "initial ajax doesnt have index 2".to_string(), 476 | }) 477 | } 478 | // println!("{:#?}",data) 479 | } 480 | 481 | fn get_primary_info_renderer(inital_data: &Value) -> Result { 482 | let contents = inital_data 483 | .get("contents") 484 | .and_then(|content| content.get("twoColumnWatchNextResults")) 485 | .and_then(|content| content.get("results")) 486 | .and_then(|content| content.get("results")) 487 | .and_then(|content| content.get("contents")) 488 | .and_then(|contents| contents.as_array()) 489 | .ok_or(ParsingError::ParsingError { 490 | cause: "cant get contents".to_string(), 491 | })?; 492 | 493 | for content in contents { 494 | if let Some(info) = content.get("videoPrimaryInfoRenderer") { 495 | return Ok(info.clone()); 496 | } 497 | } 498 | Err(ParsingError::ParsingError { 499 | cause: "could not get primary info renderer".to_string(), 500 | }) 501 | } 502 | fn get_secondary_info_renderer(inital_data: &Value) -> Result { 503 | let contents = inital_data 504 | .get("contents") 505 | .and_then(|content| content.get("twoColumnWatchNextResults")) 506 | .and_then(|content| content.get("results")) 507 | .and_then(|content| content.get("results")) 508 | .and_then(|content| content.get("contents")) 509 | .and_then(|contents| contents.as_array()) 510 | .ok_or(ParsingError::ParsingError { 511 | cause: "cant get contents".to_string(), 512 | })?; 513 | 514 | for content in contents { 515 | if let Some(info) = content.get("videoSecondaryInfoRenderer") { 516 | return Ok(info.clone()); 517 | } 518 | } 519 | Err(ParsingError::ParsingError { 520 | cause: "could not get primary info renderer".to_string(), 521 | }) 522 | } 523 | 524 | fn load_decryption_code(player_code: &str) -> Result { 525 | let decryption_func_name = YTStreamExtractor::::get_decryption_func_name(player_code) 526 | .ok_or(ParsingError::parsing_error_from_str( 527 | "Cant find decryption function", 528 | ))?; 529 | 530 | // println!("Decryption func name {}", decryption_func_name); 531 | let function_pattern = format!( 532 | "({}=function\\([a-zA-Z0-9_]+\\)\\{{.+?\\}})", 533 | decryption_func_name.replace("$", "\\$") 534 | ); 535 | 536 | let decryption_func = format!( 537 | "var {};", 538 | YTStreamExtractor::::match_group1(&function_pattern, &player_code)? 539 | ); 540 | 541 | let helper_object_name = YTStreamExtractor::::match_group1( 542 | ";([A-Za-z0-9_\\$]{2})\\...\\(", 543 | &decryption_func, 544 | )?; 545 | 546 | // print!("helper object name : {}",helper_object_name); 547 | let helper_pattern = format!( 548 | "(var {}=\\{{.+?\\}}\\}};)", 549 | helper_object_name.replace("$", "\\$") 550 | ); 551 | 552 | let helper_object = 553 | YTStreamExtractor::::match_group1(&helper_pattern, &player_code.replace("\n", ""))?; 554 | 555 | let caller_function = format!( 556 | "function {}(a){{return {}(a);}}", 557 | DECRYPTION_FUNC_NAME, decryption_func_name 558 | ); 559 | 560 | Ok(format!( 561 | "{}{}{}", 562 | helper_object, decryption_func, caller_function 563 | )) 564 | } 565 | fn load_n_param_decryption_code(player_code: &str) -> Result { 566 | let decryption_func_name = 567 | YTStreamExtractor::::get_n_param_decryption_func_name(player_code).ok_or( 568 | ParsingError::parsing_error_from_str("Cant find n_param decryption function"), 569 | )?; 570 | 571 | // println!("Decryption func name {}", decryption_func_name); 572 | 573 | let decryption_func = 574 | Self::get_n_param_decryption_function(player_code, &decryption_func_name) 575 | .ok_or(ParsingError::from("Cant get n param decryption function"))?; 576 | 577 | let caller_function = format!( 578 | "function {}(a){{return {}(a);}}", 579 | DECRYPTION_FUNC_NAME, decryption_func_name 580 | ); 581 | 582 | Ok(format!("{}{}", decryption_func, caller_function)) 583 | } 584 | 585 | fn get_decryption_func_name(player_code: &str) -> Option { 586 | // let decryption_func_name_regexes = REGEXES; 587 | // use fancy_regex::Regex; 588 | for reg in REGEXES.iter() { 589 | let rege = pcre2::bytes::Regex::new(reg).ok()?; 590 | let capture = rege.captures(player_code.as_bytes()).unwrap(); 591 | if let Some(capture) = capture { 592 | return capture.get(1).map(|m| { 593 | std::str::from_utf8(m.as_bytes()) 594 | .expect("Not utf8") 595 | .to_string() 596 | }); 597 | } 598 | } 599 | None 600 | } 601 | 602 | fn get_n_param_decryption_func_name(player_code: &str) -> Option { 603 | // let decryption_func_name_regexes = REGEXES; 604 | // use fancy_regex::Regex; 605 | for reg in [&N_PARAM_FUNC_REGEX].iter() { 606 | let rege = pcre2::bytes::Regex::new(reg).ok()?; 607 | let capture = rege.captures(player_code.as_bytes()).unwrap(); 608 | if let Some(capture) = capture { 609 | log::info!("NPARAM FUNC NAME CAPTURE {:#?}",capture); 610 | let mut function_name= capture.get(1).map(|m| { 611 | std::str::from_utf8(m.as_bytes()) 612 | .expect("Not utf8") 613 | .to_string() 614 | })?; 615 | let array_start_brace = function_name.find("["); 616 | if let Some(array_start_brace)=array_start_brace{ 617 | let array_var_name = function_name[..array_start_brace].to_string(); 618 | let order = function_name[array_start_brace+1..function_name.find("]")?].to_string(); 619 | let array_num = order.parse::().ok()?; 620 | let array_pattern = pcre2::bytes::Regex::new(&format!("var {}=\\[(.+?)\\];",array_var_name)).ok()?; 621 | 622 | let capture = array_pattern.captures(player_code.as_bytes()).unwrap(); 623 | if let Some(capture)=capture{ 624 | let array_str = capture.get(1).map(|m| { 625 | std::str::from_utf8(m.as_bytes()) 626 | .expect("Not utf8") 627 | .to_string() 628 | })?; 629 | let names = array_str.split(",").collect::>(); 630 | function_name = names.get(array_num)?.to_string(); 631 | } 632 | } 633 | return Some(function_name); 634 | 635 | } 636 | } 637 | None 638 | } 639 | 640 | fn get_n_param_decryption_function(player_code: &str, function_name: &str) -> Option { 641 | Self::get_n_param_decryption_function_name_paranthesis(player_code, function_name).or( 642 | Self::get_n_param_decryption_func_name_regex(player_code, function_name), 643 | ) 644 | } 645 | fn get_n_param_decryption_function_name_paranthesis( 646 | player_code: &str, 647 | function_name: &str, 648 | ) -> Option { 649 | let function_base = format!("{}=function", function_name); 650 | Some(format!( 651 | "{}{};", 652 | function_base, 653 | utils::match_to_closing_paranthesis(player_code, &function_base)? 654 | )) 655 | } 656 | fn get_n_param_decryption_func_name_regex( 657 | player_code: &str, 658 | function_name: &str, 659 | ) -> Option { 660 | let function_pattern = format!("{}=function(.*?}};)\n", function_name); 661 | Some(format!( 662 | "function {}{}", 663 | function_name, 664 | Self::match_group1(&function_pattern, player_code).ok()? 665 | )) 666 | } 667 | 668 | fn match_group1(reg: &str, text: &str) -> Result { 669 | let rege = pcre2::bytes::Regex::new(reg).expect("Regex is wrong"); 670 | let capture = rege.captures(text.as_bytes()).map_err(|e| e.to_string())?; 671 | if let Some(capture) = capture { 672 | return capture 673 | .get(1) 674 | .map(|m| { 675 | std::str::from_utf8(m.as_bytes()) 676 | .expect("not utf8") 677 | .to_string() 678 | }) 679 | .ok_or(ParsingError::parsing_error_from_str("group 1 not found")); 680 | } 681 | Err(ParsingError::parsing_error_from_str("regex not match")) 682 | } 683 | } 684 | 685 | impl YTStreamExtractor { 686 | pub fn get_name(&self) -> Result { 687 | let mut title = String::new(); 688 | if let Some(title_ob) = self.primary_info_renderer.get("title") { 689 | let title_ob = get_text_from_object(title_ob, false)?; 690 | if let Some(title_o) = title_ob { 691 | title = title_o; 692 | } 693 | } 694 | if title.is_empty() { 695 | if let Some(t) = self 696 | .player_response 697 | .get("videoDetails") 698 | .and_then(|t| t.get("title")) 699 | .and_then(|t| t.as_str()) 700 | { 701 | title = t.to_string(); 702 | } 703 | } 704 | if title.is_empty() { 705 | Err(ParsingError::parsing_error_from_str("Cant get title")) 706 | } else { 707 | Ok(title) 708 | } 709 | } 710 | 711 | pub fn get_description(&self, html: bool) -> Result<(String, bool), ParsingError> { 712 | if let Some(desc) = self.secondary_info_renderer.get("description") { 713 | let desc = get_text_from_object(desc, html)?; 714 | if let Some(desc) = desc { 715 | if !desc.is_empty() { 716 | return Ok((desc, true)); 717 | } 718 | } 719 | } 720 | if let Some(desc) = self 721 | .player_response 722 | .get("videoDetails") 723 | .and_then(|f| f.get("shortDescription").and_then(|f| f.as_str())) 724 | { 725 | return Ok((desc.to_string(), false)); 726 | } 727 | Err(ParsingError::parsing_error_from_str("Cant get description")) 728 | } 729 | 730 | pub fn get_video_id(&self) -> String { 731 | self.video_id.clone() 732 | } 733 | 734 | pub fn get_video_thumbnails(&self) -> Result, ParsingError> { 735 | if let Value::Object(video_details) = self 736 | .player_response 737 | .get("videoDetails") 738 | .ok_or("cant get video Details")? 739 | { 740 | if let Value::Object(thumbnail) = 741 | video_details.get("thumbnail").ok_or("cant get thumbnail")? 742 | { 743 | if let Value::Array(thumbnails) = thumbnail 744 | .get("thumbnails") 745 | .ok_or("Cant get thumbnails array")? 746 | { 747 | let mut thumbnails_str = vec![]; 748 | for thumb in thumbnails { 749 | let mut thumbnail: Thumbnail = 750 | serde_json::from_value(thumb.clone()).map_err(|e| e.to_string())?; 751 | thumbnail.url = fix_thumbnail_url(&thumbnail.url); 752 | thumbnails_str.push(thumbnail) 753 | } 754 | return Ok(thumbnails_str); 755 | } 756 | } 757 | } 758 | Err(ParsingError::parsing_error_from_str( 759 | "Cant get video thumbnails", 760 | )) 761 | } 762 | 763 | pub fn get_length(&self) -> Result { 764 | if let Some(duration) = self 765 | .player_response 766 | .get("videoDetails") 767 | .and_then(|f| f.get("lengthSeconds")) 768 | .and_then(|f| f.as_str()) 769 | { 770 | if let Ok(duration) = duration.parse::() { 771 | return Ok(duration); 772 | } 773 | } 774 | if let Some(duration_ms) = self 775 | .player_response 776 | .get("streamingData") 777 | .and_then(|f| f.get("formats")) 778 | .and_then(|f| f.as_array()) 779 | .and_then(|f| f.get(0)) 780 | .and_then(|f| f.get("approxDurationMs")) 781 | .and_then(|f| f.as_str()) 782 | { 783 | if let Ok(duration) = duration_ms.parse::() { 784 | return Ok(duration / 1000); 785 | } 786 | } 787 | 788 | Err(ParsingError::parsing_error_from_str("Cant get length")) 789 | } 790 | 791 | pub fn get_view_count(&self) -> Result { 792 | let mut views = String::new(); 793 | if let Some(vc) = self 794 | .primary_info_renderer 795 | .get("viewCount") 796 | .and_then(|f| f.get("videoViewCountRenderer")) 797 | .and_then(|f| f.get("viewCount")) 798 | { 799 | views = get_text_from_object(vc, false)?.unwrap_or("".to_string()); 800 | } 801 | if views.is_empty() { 802 | if let Some(vc) = self 803 | .player_response 804 | .get("videoDetails") 805 | .and_then(|f| f.get("viewCount")) 806 | .and_then(|f| f.as_str()) 807 | { 808 | views = vc.to_string(); 809 | } 810 | } 811 | if !views.is_empty() { 812 | if views.to_ascii_lowercase().contains("no views") { 813 | return Ok(0); 814 | } else { 815 | if let Ok(views) = remove_non_digit_chars::(&views) { 816 | return Ok(views); 817 | } 818 | } 819 | } 820 | // println!("{}",views); 821 | Err(ParsingError::parsing_error_from_str("Cant get view count")) 822 | } 823 | 824 | pub fn get_like_count(&self) -> Result { 825 | let mut like_string = String::new(); 826 | if let Some(likes) = self 827 | .primary_info_renderer 828 | .get("sentimentBar") 829 | .and_then(|f| f.get("sentimentBarRenderer")) 830 | .and_then(|f| f.get("tooltip")) 831 | .and_then(|f| f.as_str()) 832 | { 833 | if let Some(lks) = likes.split("/").next() { 834 | like_string = lks.to_string(); 835 | } 836 | } 837 | if like_string.is_empty() { 838 | if let Some(allow_ratings) = self 839 | .player_response 840 | .get("videoDetails") 841 | .and_then(|f| f.get("allowRatings")) 842 | .and_then(|f| f.as_bool()) 843 | { 844 | if allow_ratings { 845 | return Err(ParsingError::parsing_error_from_str( 846 | "Ratings are enabled even though the like button is missing", 847 | )); 848 | } else { 849 | return Ok(-1); 850 | } 851 | } 852 | } else { 853 | if let Ok(likes) = remove_non_digit_chars::(&like_string) { 854 | return Ok(likes); 855 | } 856 | } 857 | Err(ParsingError::parsing_error_from_str( 858 | "could not get like count", 859 | )) 860 | } 861 | 862 | pub fn get_dislike_count(&self) -> Result { 863 | let mut like_string = String::new(); 864 | if let Some(likes) = self 865 | .primary_info_renderer 866 | .get("sentimentBar") 867 | .and_then(|f| f.get("sentimentBarRenderer")) 868 | .and_then(|f| f.get("tooltip")) 869 | .and_then(|f| f.as_str()) 870 | { 871 | if let Some(lks) = likes.split("/").nth(1) { 872 | like_string = lks.to_string(); 873 | } 874 | } 875 | if like_string.is_empty() { 876 | if let Some(allow_ratings) = self 877 | .player_response 878 | .get("videoDetails") 879 | .and_then(|f| f.get("allowRatings")) 880 | .and_then(|f| f.as_bool()) 881 | { 882 | if allow_ratings { 883 | return Err(ParsingError::parsing_error_from_str( 884 | "Ratings are enabled even though the dislike button is missing", 885 | )); 886 | } else { 887 | return Ok(-1); 888 | } 889 | } 890 | } else { 891 | if let Ok(likes) = remove_non_digit_chars::(&like_string) { 892 | return Ok(likes); 893 | } 894 | } 895 | Err(ParsingError::parsing_error_from_str( 896 | "could not get dislike count", 897 | )) 898 | } 899 | 900 | pub fn get_uploader_url(&self) -> Result { 901 | if let Some(nav_end) = self 902 | .secondary_info_renderer 903 | .get("owner") 904 | .and_then(|f| f.get("videoOwnerRenderer")) 905 | .and_then(|f| f.get("navigationEndpoint")) 906 | { 907 | let uploader_url = get_url_from_navigation_endpoint(nav_end)?; 908 | if !uploader_url.is_empty() { 909 | return Ok(uploader_url); 910 | } 911 | } 912 | if let Some(uploader_id) = self 913 | .player_response 914 | .get("videoDetails") 915 | .and_then(|f| f.get("channelId")) 916 | .and_then(|f| f.as_str()) 917 | { 918 | return Ok(format!("https://www.youtube.com/channel/{}", uploader_id)); 919 | } 920 | Err(ParsingError::parsing_error_from_str( 921 | "Cant get uploader url", 922 | )) 923 | } 924 | 925 | pub fn get_uploader_name(&self) -> Result { 926 | let mut uploader_name = String::new(); 927 | if let Some(uploader) = self 928 | .secondary_info_renderer 929 | .get("owner") 930 | .and_then(|f| f.get("videoOwnerRenderer")) 931 | .and_then(|f| f.get("title")) 932 | { 933 | if let Some(uploader) = get_text_from_object(uploader, false)? { 934 | uploader_name = uploader; 935 | } 936 | } 937 | if uploader_name.is_empty() { 938 | if let Some(author) = self 939 | .player_response 940 | .get("videoDetails") 941 | .and_then(|f| f.get("author")) 942 | .and_then(|f| f.as_str()) 943 | { 944 | uploader_name = author.to_owned(); 945 | } 946 | } 947 | 948 | if uploader_name.is_empty() { 949 | Err(ParsingError::parsing_error_from_str( 950 | "Cant get uploader name", 951 | )) 952 | } else { 953 | Ok(uploader_name) 954 | } 955 | } 956 | 957 | pub fn get_uploader_avatar_url(&self) -> Result, ParsingError> { 958 | let mut thumbnails = vec![]; 959 | if let Some(thumbs) = self 960 | .secondary_info_renderer 961 | .get("owner") 962 | .and_then(|f| f.get("videoOwnerRenderer")) 963 | .and_then(|f| f.get("thumbnail")) 964 | .and_then(|f| f.get("thumbnails")) 965 | .and_then(|f| f.as_array()) 966 | { 967 | for thumb in thumbs { 968 | if let Ok(mut thumb) = serde_json::from_value::(thumb.clone()) { 969 | thumb.url = fix_thumbnail_url(&thumb.url); 970 | thumbnails.push(thumb); 971 | } 972 | } 973 | } 974 | Ok(thumbnails) 975 | } 976 | 977 | // pub fn is_live(&self)->Result{ 978 | // if let Some(format) = self.player_response.get("streamingData").and_then(|f|f.get(FORMATS)){ 979 | // return Ok(true); 980 | // }else if let Some(ps)= self.player_args.get("ps").and_then(|f|f.as_str()){ 981 | // println!("{}",ps); 982 | // if ps=="live"{ 983 | // return Ok(true) 984 | // } 985 | // } 986 | // Ok(false) 987 | // } 988 | } 989 | 990 | impl YTStreamExtractor { 991 | pub async fn get_video_stream(&self) -> Result, ParsingError> { 992 | let mut video_streams = vec![]; 993 | for entry in YTStreamExtractor::::get_itags( 994 | FORMATS, 995 | ItagType::Video, 996 | &self.player_response, 997 | &self.player_decryption_code, 998 | &self.player_n_param_decryption_code, &self.downloader 999 | 1000 | ).await? { 1001 | let itag = entry.1; 1002 | video_streams.push(StreamItem { 1003 | url: Some(entry.0), 1004 | ..itag 1005 | }); 1006 | } 1007 | Ok(video_streams) 1008 | } 1009 | 1010 | pub async fn get_video_only_stream(&self) -> Result, ParsingError> { 1011 | let mut video_streams = vec![]; 1012 | for entry in YTStreamExtractor::::get_itags( 1013 | ADAPTIVE_FORMATS, 1014 | ItagType::VideoOnly, 1015 | &self.player_response, 1016 | &self.player_decryption_code, 1017 | &self.player_n_param_decryption_code, 1018 | &self.downloader 1019 | ).await? { 1020 | let itag = entry.1; 1021 | video_streams.push(StreamItem { 1022 | url: Some(entry.0), 1023 | ..itag 1024 | }); 1025 | } 1026 | Ok(video_streams) 1027 | } 1028 | 1029 | pub async fn get_audio_streams(&self) -> Result, ParsingError> { 1030 | let mut audio_streams = vec![]; 1031 | for entry in YTStreamExtractor::::get_itags( 1032 | ADAPTIVE_FORMATS, 1033 | ItagType::Audio, 1034 | &self.player_response, 1035 | &self.player_decryption_code, 1036 | &self.player_n_param_decryption_code, &self.downloader 1037 | 1038 | ).await? { 1039 | let itag = entry.1; 1040 | audio_streams.push(StreamItem { 1041 | url: Some(entry.0), 1042 | ..itag 1043 | }); 1044 | } 1045 | 1046 | Ok(audio_streams) 1047 | } 1048 | 1049 | pub fn get_related(&self) -> Result, ParsingError> { 1050 | let results = (|| { 1051 | self.initial_data 1052 | .get("contents")? 1053 | .get("twoColumnWatchNextResults")? 1054 | .get("secondaryResults")? 1055 | .get("secondaryResults")? 1056 | .get("results")? 1057 | .as_array() 1058 | .cloned() 1059 | })() 1060 | .unwrap_or_default(); 1061 | use crate::youtube_extractor::search_extractor::YTSearchExtractor; 1062 | let items = YTSearchExtractor::::collect_streams_from(&results); 1063 | items 1064 | } 1065 | } 1066 | -------------------------------------------------------------------------------- /src/youtube_extractor/stream_info_item_extractor.rs: -------------------------------------------------------------------------------- 1 | use crate::utils::utils::*; 2 | use crate::youtube_extractor::error::ParsingError; 3 | use crate::youtube_extractor::stream_extractor::Thumbnail; 4 | use serde_json::{Map, Value}; 5 | use std::convert::TryInto; 6 | 7 | #[derive(Clone, PartialEq)] 8 | pub struct YTStreamInfoItemExtractor { 9 | pub video_info: Map, 10 | } 11 | impl YTStreamInfoItemExtractor { 12 | pub fn get_name(&self) -> Result { 13 | if let Some(title) = self.video_info.get("title") { 14 | let name = get_text_from_object(title, false)?; 15 | if let Some(name) = name { 16 | if !name.is_empty() { 17 | return Ok(name); 18 | } 19 | } 20 | } 21 | Err(ParsingError::from("Cannot get name")) 22 | } 23 | 24 | pub fn is_ad(&self) -> Result { 25 | Ok(self.is_premium_video()? 26 | || self.get_name()? == "[Private video]" 27 | || self.get_name()? == "[Deleted video]") 28 | } 29 | 30 | pub fn video_id(&self) -> Result { 31 | Ok(self 32 | .video_info 33 | .get("videoId") 34 | .ok_or("video id not found")? 35 | .as_str() 36 | .ok_or("videoid not string")? 37 | .to_string()) 38 | } 39 | 40 | pub fn is_premium_video(&self) -> Result { 41 | let badges = self 42 | .video_info 43 | .get("badges") 44 | .unwrap_or(&Value::Null) 45 | .as_array(); 46 | if let Some(badges) = badges { 47 | for badge in badges { 48 | if badge 49 | .get("metadataBadgeRenderer") 50 | .ok_or("metadataBadgeRenderer not found")? 51 | .get("label") 52 | .unwrap_or(&Value::Null) 53 | .as_str() 54 | .unwrap_or("") 55 | == "Premium" 56 | { 57 | return Ok(true); 58 | } 59 | } 60 | } 61 | Ok(false) 62 | } 63 | 64 | pub fn get_url(&self) -> Result { 65 | let id = self.video_id()?; 66 | Ok(format!("https://www.youtube.com/watch?v={}", id)) 67 | } 68 | 69 | pub fn is_live(&self) -> Result { 70 | let badges = self 71 | .video_info 72 | .get("badges") 73 | .unwrap_or(&Value::Null) 74 | .as_array(); 75 | 76 | if let Some(badges) = badges { 77 | for badge in badges { 78 | if badge 79 | .get("metadataBadgeRenderer") 80 | .ok_or("metadataBadgeRenderer not found")? 81 | .get("label") 82 | .unwrap_or(&Value::Null) 83 | .as_str() 84 | .unwrap_or("") 85 | == "LIVE NOW" 86 | { 87 | return Ok(true); 88 | } 89 | } 90 | } 91 | 92 | let style = self 93 | .video_info 94 | .get("thumbnailOverlays") 95 | .unwrap_or(&Value::Null) 96 | .get(0) 97 | .unwrap_or(&Value::Null) 98 | .get("thumbnailOverlayTimeStatusRenderer") 99 | .unwrap_or(&Value::Null) 100 | .get("style") 101 | .unwrap_or(&Value::Null) 102 | .as_str() 103 | .unwrap_or(""); 104 | if style.eq_ignore_ascii_case("LIVE") { 105 | return Ok(true); 106 | } 107 | 108 | Ok(false) 109 | } 110 | 111 | pub fn get_textual_duration(&self) -> Result { 112 | if self.is_live()? { 113 | return Ok("LIVE".to_string()); 114 | } 115 | let mut duration = get_text_from_object( 116 | self.video_info 117 | .get("lengthText") 118 | .ok_or("Cant get lengthText")?, 119 | false, 120 | )?; 121 | if duration.is_none() || duration.clone().unwrap_or_default().is_empty() { 122 | for thumbnail_overlay in self 123 | .video_info 124 | .get("thumbnailOverlays") 125 | .unwrap_or(&Value::Null) 126 | .as_array() 127 | .unwrap_or(&vec![]) 128 | { 129 | if let Some(tr_renderer) = 130 | thumbnail_overlay.get("thumbnailOverlayTimeStatusRenderer") 131 | { 132 | duration = get_text_from_object( 133 | tr_renderer.get("text").unwrap_or(&Value::Null), 134 | false, 135 | )?; 136 | } 137 | } 138 | } 139 | if duration.is_none() || duration.clone().unwrap_or_default().is_empty() { 140 | Err(ParsingError::from("Cant get duration")) 141 | } else { 142 | Ok(duration.unwrap_or_default()) 143 | } 144 | } 145 | 146 | pub fn get_duration(&self) -> Result { 147 | if self.is_live()? { 148 | return Ok(-1); 149 | } 150 | let mut duration = get_text_from_object( 151 | self.video_info 152 | .get("lengthText") 153 | .ok_or("Cant get lengthText")?, 154 | false, 155 | )?; 156 | if duration.is_none() || duration.clone().unwrap_or_default().is_empty() { 157 | for thumbnail_overlay in self 158 | .video_info 159 | .get("thumbnailOverlays") 160 | .unwrap_or(&Value::Null) 161 | .as_array() 162 | .unwrap_or(&vec![]) 163 | { 164 | if let Some(tr_renderer) = 165 | thumbnail_overlay.get("thumbnailOverlayTimeStatusRenderer") 166 | { 167 | duration = get_text_from_object( 168 | tr_renderer.get("text").unwrap_or(&Value::Null), 169 | false, 170 | )?; 171 | } 172 | } 173 | } 174 | if duration.is_none() || duration.clone().unwrap_or_default().is_empty() { 175 | Err(ParsingError::from("Cant get duration")) 176 | } else { 177 | Ok(remove_non_digit_chars::(&duration.unwrap_or_default()) 178 | .map_err(|f| ParsingError::from(f.to_string()))?) 179 | } 180 | } 181 | 182 | pub fn get_uploader_name(&self) -> Result { 183 | let mut name = get_text_from_object( 184 | self.video_info 185 | .get("longBylineText") 186 | .unwrap_or(&Value::Null), 187 | false, 188 | )? 189 | .unwrap_or_default(); 190 | if name.is_empty() { 191 | name = get_text_from_object( 192 | self.video_info.get("ownerText").unwrap_or(&Value::Null), 193 | false, 194 | )? 195 | .unwrap_or_default(); 196 | 197 | if name.is_empty() { 198 | name = get_text_from_object( 199 | self.video_info 200 | .get("shortBylineText") 201 | .unwrap_or(&Value::Null), 202 | false, 203 | )? 204 | .unwrap_or_default(); 205 | 206 | if name.is_empty() { 207 | return Err(ParsingError::from("Cant get uploader name")); 208 | } 209 | } 210 | } 211 | 212 | Ok(name) 213 | } 214 | 215 | pub fn get_uploader_url(&self) -> Result { 216 | let mut url = get_url_from_navigation_endpoint( 217 | self.video_info 218 | .get("longBylineText") 219 | .unwrap_or(&Value::Null) 220 | .get("runs") 221 | .unwrap_or(&Value::Null) 222 | .get(0) 223 | .unwrap_or(&Value::Null) 224 | .get("navigationEndpoint") 225 | .unwrap_or(&Value::Null), 226 | ); 227 | if url.is_err() || url.clone().unwrap_or_default().is_empty() { 228 | url = get_url_from_navigation_endpoint( 229 | self.video_info 230 | .get("ownerText") 231 | .unwrap_or(&Value::Null) 232 | .get("runs") 233 | .unwrap_or(&Value::Null) 234 | .get(0) 235 | .unwrap_or(&Value::Null) 236 | .get("navigationEndpoint") 237 | .unwrap_or(&Value::Null), 238 | ); 239 | if url.is_err() || url.clone().unwrap_or_default().is_empty() { 240 | url = get_url_from_navigation_endpoint( 241 | self.video_info 242 | .get("shortBylineText") 243 | .unwrap_or(&Value::Null) 244 | .get("runs") 245 | .unwrap_or(&Value::Null) 246 | .get(0) 247 | .unwrap_or(&Value::Null) 248 | .get("navigationEndpoint") 249 | .unwrap_or(&Value::Null), 250 | ); 251 | 252 | if url.is_err() || url.clone().unwrap_or_default().is_empty() { 253 | return Err(ParsingError::from("Cant get uploader url")); 254 | } 255 | } 256 | } 257 | url 258 | } 259 | 260 | pub fn get_textual_upload_date(&self) -> Result { 261 | if self.is_live()? { 262 | return Err(ParsingError::from("live video has no upload date")); 263 | } 264 | let pt = get_text_from_object( 265 | self.video_info 266 | .get("publishedTimeText") 267 | .unwrap_or(&Value::Null), 268 | false, 269 | )?; 270 | Ok(pt.ok_or("Cant get upload date")?) 271 | } 272 | 273 | pub fn get_textual_view_count(&self) -> Result { 274 | if self.is_premium_video()? || self.video_info.contains_key("topStandaloneBadge") { 275 | return Ok("".to_string()); 276 | } 277 | if let Some(viewc) = self.video_info.get("viewCountText") { 278 | let view_count = get_text_from_object(viewc, false)?.unwrap_or_default(); 279 | if view_count.to_ascii_lowercase().contains("no views") { 280 | return Ok("no views".to_string()); 281 | } else if view_count.to_ascii_lowercase().contains("recommended") { 282 | return Err(ParsingError::from("views hidden")); 283 | } else { 284 | return Ok(view_count); 285 | } 286 | } 287 | 288 | Err(ParsingError::from("Cant get view count")) 289 | } 290 | 291 | pub fn get_view_count(&self) -> Result { 292 | if self.is_premium_video()? || self.video_info.contains_key("topStandaloneBadge") { 293 | return Ok(-1); 294 | } 295 | if let Some(viewc) = self.video_info.get("viewCountText") { 296 | let view_count = get_text_from_object(viewc, false)?.unwrap_or_default(); 297 | if view_count.to_ascii_lowercase().contains("no views") { 298 | return Ok(0); 299 | } else if view_count.to_ascii_lowercase().contains("recommended") { 300 | return Ok(-1); 301 | } else { 302 | return Ok(remove_non_digit_chars::(&view_count) 303 | .map_err(|er| ParsingError::from(er.to_string()))?); 304 | } 305 | } 306 | 307 | Err(ParsingError::from("Cant get view count")) 308 | } 309 | 310 | pub fn get_thumbnails(&self) -> Result, ParsingError> { 311 | let mut thumbnails = vec![]; 312 | for thumb in self 313 | .video_info 314 | .get("thumbnail") 315 | .ok_or("No thumbnail")? 316 | .get("thumbnails") 317 | .ok_or("no thumbnails")? 318 | .as_array() 319 | .ok_or("thumbnails array")? 320 | { 321 | // println!("{:#?}",thumb); 322 | if let Ok(thumb) = serde_json::from_value(thumb.to_owned()) { 323 | thumbnails.push(thumb) 324 | } 325 | } 326 | Ok(thumbnails) 327 | } 328 | 329 | pub fn get_uploader_thumbnails(&self) -> Result, ParsingError> { 330 | let mut thumbnails = vec![]; 331 | for thumb in self 332 | .video_info 333 | .get("channelThumbnailSupportedRenderers") 334 | .ok_or("no channel thumbnail")? 335 | .get("channelThumbnailWithLinkRenderer") 336 | .ok_or("no channel thumbnail")? 337 | .get("thumbnail") 338 | .ok_or("No thumbnail")? 339 | .get("thumbnails") 340 | .ok_or("no thumbnails")? 341 | .as_array() 342 | .ok_or("thumbnails array")? 343 | { 344 | // println!("{:#?}",thumb); 345 | if let Ok(thumb) = serde_json::from_value(thumb.to_owned()) { 346 | thumbnails.push(thumb) 347 | } 348 | } 349 | Ok(thumbnails) 350 | } 351 | } 352 | -------------------------------------------------------------------------------- /src/youtube_extractor/trending_extractor.rs: -------------------------------------------------------------------------------- 1 | use crate::downloader_trait::Downloader; 2 | use crate::youtube_extractor::error::ParsingError; 3 | use crate::youtube_extractor::stream_extractor::HARDCODED_CLIENT_VERSION; 4 | use crate::youtube_extractor::stream_info_item_extractor::YTStreamInfoItemExtractor; 5 | use serde_json::Value; 6 | use std::collections::HashMap; 7 | 8 | #[derive(Clone, PartialEq)] 9 | pub struct YTTrendingExtractor { 10 | initial_data: Value, 11 | } 12 | 13 | impl YTTrendingExtractor { 14 | async fn get_initial_data(downloader: &D) -> Result { 15 | let url = format!("https://www.youtube.com/feed/trending?pbj=1"); 16 | let mut headers = HashMap::new(); 17 | headers.insert("X-YouTube-Client-Name".to_string(), "1".to_string()); 18 | headers.insert( 19 | "X-YouTube-Client-Version".to_string(), 20 | HARDCODED_CLIENT_VERSION.to_string(), 21 | ); 22 | let url = format!("{}&pbj=1", url); 23 | let data = downloader.download_with_header(&url, headers).await?; 24 | let mut json = 25 | serde_json::from_str::(&data).map_err(|e| ParsingError::from(e.to_string()))?; 26 | Ok(json 27 | .get_mut(1) 28 | .ok_or("No index 1")? 29 | .get_mut("response") 30 | .ok_or("No response")? 31 | .take()) 32 | } 33 | 34 | pub async fn new(downloader: D) -> Result { 35 | let initial_data = YTTrendingExtractor::get_initial_data(&downloader).await?; 36 | Ok(Self { initial_data }) 37 | } 38 | } 39 | 40 | impl YTTrendingExtractor { 41 | pub fn get_videos(&self) -> Result, ParsingError> { 42 | let item_section_renderers = (|| { 43 | self.initial_data 44 | .get("contents")? 45 | .get("twoColumnBrowseResultsRenderer")? 46 | .get("tabs")? 47 | .get(0)? 48 | .get("tabRenderer")? 49 | .get("content")? 50 | .get("sectionListRenderer")? 51 | .get("contents")? 52 | .as_array() 53 | })() 54 | .ok_or("No item sections")?; 55 | let mut videos = vec![]; 56 | for item_section in item_section_renderers { 57 | let shelf_content = (|| { 58 | item_section 59 | .get("itemSectionRenderer")? 60 | .get("contents")? 61 | .get(0)? 62 | .get("shelfRenderer")? 63 | .get("content")? 64 | .get("expandedShelfContentsRenderer")? 65 | .get("items")? 66 | .as_array() 67 | })(); 68 | if let Some(shelf_content) = shelf_content { 69 | for ul in shelf_content { 70 | if let Some(videoRenderer) = 71 | ul.get("videoRenderer").unwrap_or(&Value::Null).as_object() 72 | { 73 | videos.push(YTStreamInfoItemExtractor { 74 | video_info: videoRenderer.clone(), 75 | }) 76 | } 77 | } 78 | } 79 | } 80 | Ok(videos) 81 | } 82 | } 83 | --------------------------------------------------------------------------------