├── rsget_cli ├── .gitignore ├── Cargo.toml └── src │ └── main.rs ├── rsget_lib ├── src │ ├── plugins │ │ ├── default.rs │ │ ├── mod.rs │ │ ├── drdk.rs │ │ ├── dlive.rs │ │ ├── douyin.rs │ │ ├── bilibili.rs │ │ ├── inke.rs │ │ ├── twitch.rs │ │ ├── vlive.rs │ │ ├── douyu.rs │ │ ├── afreeca.rs │ │ ├── huya.rs │ │ └── tiktok.rs │ ├── utils │ │ ├── mod.rs │ │ ├── downloaders.rs │ │ ├── sites.rs │ │ └── error.rs │ └── lib.rs ├── .gitignore └── Cargo.toml ├── detective ├── tiktok │ └── tiktok-video ├── twitch │ └── m3u8_master_playlist └── main.org ├── .gitignore ├── README.md ├── Cargo.toml ├── .gitlab-ci.yml ├── .github └── workflows │ └── ci.yml ├── LICENSE └── stream_lib ├── src ├── download_stream.rs ├── lib.rs ├── error.rs └── hls │ ├── watch.rs │ ├── mod.rs │ └── named_watch.rs ├── Cargo.toml ├── examples └── hls_download.rs └── README.md /rsget_cli/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | /target/ 3 | **/*.rs.bk 4 | -------------------------------------------------------------------------------- /rsget_lib/src/plugins/default.rs: -------------------------------------------------------------------------------- 1 | use super::*; 2 | -------------------------------------------------------------------------------- /rsget_lib/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | /target/ 3 | **/*.rs.bk 4 | Cargo.lock 5 | -------------------------------------------------------------------------------- /rsget_lib/src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | //pub mod downloaders; 2 | pub mod error; 3 | pub mod sites; 4 | -------------------------------------------------------------------------------- /detective/tiktok/tiktok-video: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Erk-/rsget/HEAD/detective/tiktok/tiktok-video -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.flv 3 | *.mp4 4 | #*# 5 | .#* 6 | 7 | /target 8 | **/*.rs.bk 9 | Cargo.lock 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rsget 2 | 3 | [![Build Status](https://github.com/erk-/rsget/workflows/Test/badge.svg)](https://github.com/erk-/rsget/actions) 4 | 5 | Rsget is a download tool for streams on websites. 6 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "rsget_lib", 4 | "rsget_cli", 5 | "stream_lib", 6 | ] 7 | resolver = "2" 8 | 9 | [profile.release] 10 | panic = "abort" 11 | lto = true 12 | -------------------------------------------------------------------------------- /rsget_lib/src/plugins/mod.rs: -------------------------------------------------------------------------------- 1 | mod afreeca; 2 | mod bilibili; 3 | mod dlive; 4 | mod twitch; 5 | mod drdk; 6 | mod vlive; 7 | 8 | pub use afreeca::Afreeca; 9 | pub use bilibili::Bilibili; 10 | pub use dlive::DLive; 11 | pub use drdk::Drdk; 12 | pub use twitch::Twitch; 13 | pub use vlive::Vlive; 14 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: "rust:latest" 2 | 3 | stages: 4 | - build 5 | 6 | build: 7 | stage: build 8 | script: 9 | - cd rsget_cli 10 | - cargo build --release 11 | cache: 12 | paths: 13 | - rsget_cli/target/ 14 | artifacts: 15 | paths: 16 | - rsget_cli/target/release/rsget_cli -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | name: Test 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - name: Checkout sources 12 | uses: actions/checkout@v2 13 | 14 | - name: Install stable toolchain 15 | id: toolchain 16 | uses: actions-rs/toolchain@v1 17 | with: 18 | toolchain: stable 19 | profile: minimal 20 | override: true 21 | 22 | - name: Run cargo test 23 | uses: actions-rs/cargo@v1 24 | with: 25 | command: test 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018-2022 Valdemar Erk 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any purpose 4 | with or without fee is hereby granted, provided that the above copyright notice 5 | and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 8 | REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND 9 | FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 10 | INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS 11 | OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 12 | TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF 13 | THIS SOFTWARE. -------------------------------------------------------------------------------- /stream_lib/src/download_stream.rs: -------------------------------------------------------------------------------- 1 | use bytes::Bytes; 2 | use futures_core::stream::Stream; 3 | use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; 4 | 5 | /// This struct implments a stream that is used to 6 | /// received data from chunked and hls streams. 7 | #[derive(Debug)] 8 | pub struct DownloadStream { 9 | rx: UnboundedReceiver, 10 | } 11 | 12 | impl DownloadStream { 13 | pub(crate) fn new() -> (Self, UnboundedSender) { 14 | let (tx, rx) = unbounded_channel(); 15 | (DownloadStream { rx }, tx) 16 | } 17 | } 18 | 19 | impl Stream for DownloadStream { 20 | type Item = Event; 21 | 22 | fn poll_next( 23 | mut self: std::pin::Pin<&mut Self>, 24 | cx: &mut std::task::Context<'_>, 25 | ) -> std::task::Poll> { 26 | self.rx.poll_recv(cx) 27 | } 28 | } 29 | 30 | #[derive(Debug)] 31 | pub enum Event { 32 | /// Bytes to be written to 33 | Bytes { 34 | bytes: Bytes, 35 | }, 36 | End, 37 | Error { 38 | error: crate::Error, 39 | }, 40 | } 41 | -------------------------------------------------------------------------------- /stream_lib/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "stream_lib" 3 | version = "0.5.2" 4 | authors = ["Valdemar Erk "] 5 | description = "Tool to download differnt types of streams" 6 | license = "ISC" 7 | edition = "2021" 8 | documentation = "https://docs.rs/stream_lib" 9 | homepage = "https://github.com/Erk-/rsget" 10 | repository = "https://github.com/Erk-/rsget/tree/master/stream_lib" 11 | 12 | [dependencies] 13 | hls_m3u8 = "0.4.1" 14 | reqwest = { version = "0.12", default-features = false, features = ["stream"] } 15 | tracing = "0.1.40" 16 | url = "2.5.0" 17 | futures-util = "0.3.30" 18 | tokio = { version = "1.38.0", default-features = false, features = ["rt", "sync", "time"] } 19 | patricia_tree = "0.8.0" 20 | futures-core = "0.3.30" 21 | bytes = "1.5.0" 22 | 23 | [dev-dependencies] 24 | tracing-subscriber = "0.3.18" 25 | tokio = { version = "1.38", default-features = false, features = ["fs", "rt", "sync", "time", "macros"] } 26 | 27 | [features] 28 | # Default to rustls so we don't pull in openssl 29 | default = ["rustls-tls"] 30 | rustls-tls = [ 31 | "reqwest/rustls-tls", 32 | "reqwest/rustls-tls-webpki-roots", 33 | ] 34 | native-tls = [ 35 | "reqwest/native-tls", 36 | ] 37 | -------------------------------------------------------------------------------- /rsget_cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rsget" 3 | version = "0.1.5" 4 | authors = ["Valdemar Erk "] 5 | description = "Tool to get information about and download livestreams" 6 | license = "ISC" 7 | edition = "2021" 8 | documentation = "https://docs.rs/rsget" 9 | homepage = "https://github.com/Erk-/rsget" 10 | repository = "https://github.com/Erk-/rsget/tree/master/rsget_cli" 11 | 12 | [dependencies] 13 | rsget_lib = { version = "0.3", path = "../rsget_lib" } 14 | stream_lib = { version = "0.5", path = "../stream_lib" } 15 | tracing = "0.1" 16 | tracing-subscriber = "0.3.18" 17 | clap = { version = "4.4.11", features = ["derive"] } 18 | tokio = { version = "1", features = ["fs", "rt-multi-thread", "io-util", "io-std"] } 19 | reqwest = { version = "0.12", default-features = false} 20 | indicatif = "0.17.7" 21 | futures-util = "0.3.30" 22 | 23 | [features] 24 | # Default to rustls so we don't pull in openssl 25 | default = ["rustls-tls"] 26 | rustls-tls = [ 27 | "reqwest/rustls-tls", 28 | "reqwest/rustls-tls-webpki-roots", 29 | ] 30 | native-tls = [ 31 | "reqwest/native-tls", 32 | "reqwest/rustls-tls-native-roots", 33 | ] 34 | 35 | [[bin]] 36 | name = "rsget" 37 | path = "src/main.rs" 38 | -------------------------------------------------------------------------------- /stream_lib/examples/hls_download.rs: -------------------------------------------------------------------------------- 1 | use futures_util::StreamExt as _; 2 | use reqwest::Client; 3 | use stream_lib::Event; 4 | use tokio::io::AsyncWriteExt; 5 | 6 | /// Write buffer 7 | pub const WRITE_SIZE: usize = 131_072; 8 | 9 | #[tokio::main] 10 | async fn main() -> Result<(), Box> { 11 | tracing_subscriber::fmt::init(); 12 | let args = std::env::args().collect::>(); 13 | let url = args.get(1).expect("Pass a url as the first argument"); 14 | 15 | let http = Client::new(); 16 | let req = http.get(url).build()?; 17 | let mut dl = stream_lib::download_hls(http, req, None); 18 | 19 | let mut file = tokio::io::BufWriter::with_capacity( 20 | WRITE_SIZE, 21 | tokio::fs::File::create("./test.mp4").await?, 22 | ); 23 | 24 | while let Some(event) = dl.next().await { 25 | match event { 26 | Event::Bytes { bytes } => { 27 | file.write_all(&bytes).await?; 28 | } 29 | Event::End => break, 30 | Event::Error { error } => { 31 | eprintln!("Encounted error: {}", error); 32 | break; 33 | } 34 | } 35 | } 36 | Ok(()) 37 | } 38 | -------------------------------------------------------------------------------- /rsget_lib/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rsget_lib" 3 | version = "0.3.0" 4 | authors = ["Valdemar Erk "] 5 | description = "Library to get information about and download livestreams" 6 | license = "ISC" 7 | edition = "2021" 8 | documentation = "https://docs.rs/rsget_lib" 9 | homepage = "https://github.com/Erk-/rsget" 10 | repository = "https://github.com/Erk-/rsget/tree/master/rsget_lib" 11 | 12 | [dependencies] 13 | tracing = "0.1" 14 | serde = "^1.0" 15 | serde_derive = "1.0" 16 | serde_json = "1.0" 17 | serde_urlencoded = "0.7" 18 | regex = "1.10" 19 | http = "0.2.11" 20 | chrono = "0.4" 21 | hls_m3u8 = "0.4" 22 | reqwest = { version = "0.12", default-features = false, features = ["json"] } 23 | rand = { version ="0.8", features = ["small_rng"] } 24 | async-trait = "0.1" 25 | webbrowser = "0.8" 26 | 27 | [dependencies.stream_lib] 28 | default-features = false 29 | version = "0.5.0" 30 | path = "../stream_lib" 31 | features = [] 32 | 33 | [features] 34 | # Default to rustls so we don't pull in openssl 35 | default = ["rustls-tls"] 36 | rustls-tls = [ 37 | "reqwest/rustls-tls", 38 | "reqwest/rustls-tls-webpki-roots", 39 | ] 40 | native-tls = [ 41 | "reqwest/native-tls", 42 | "reqwest/rustls-tls-native-roots", 43 | ] 44 | -------------------------------------------------------------------------------- /stream_lib/README.md: -------------------------------------------------------------------------------- 1 | # Stream Lib 2 | 3 | This library makes it possible to download various types of video streams. 4 | Currently it supports HLS and chunked http streams. 5 | 6 | ## Example 7 | 8 | ```rust 9 | use futures_util::StreamExt as _; 10 | use reqwest::Client; 11 | use stream_lib::Event; 12 | use tokio::io::AsyncWriteExt; 13 | 14 | /// Write buffer 15 | pub const WRITE_SIZE: usize = 131_072; 16 | 17 | #[tokio::main] 18 | async fn main() -> Result<(), Box> { 19 | tracing_subscriber::fmt::init(); 20 | let args = std::env::args().collect::>(); 21 | let url = args.get(1).expect("Pass a url as the first argument"); 22 | 23 | let http = Client::new(); 24 | let req = http.get(url).build()?; 25 | let mut dl = stream_lib::download_hls(http, req, None); 26 | 27 | let mut file = tokio::io::BufWriter::with_capacity( 28 | WRITE_SIZE, 29 | tokio::fs::File::create("./example.mp4").await?, 30 | ); 31 | 32 | while let Some(event) = dl.next().await { 33 | match event { 34 | Event::Bytes { bytes } => { 35 | file.write_all(&bytes).await?; 36 | } 37 | Event::End => break, 38 | Event::Error { error } => { 39 | eprintln!("Encounted error: {}", error); 40 | break; 41 | } 42 | } 43 | } 44 | Ok(()) 45 | } 46 | ``` 47 | -------------------------------------------------------------------------------- /stream_lib/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! This is a small tool to download streams 2 | //! It currently supports chunked streams and HLS. 3 | 4 | mod download_stream; 5 | mod error; 6 | mod hls; 7 | 8 | use std::time::Duration; 9 | 10 | pub use crate::download_stream::{DownloadStream, Event}; 11 | pub use crate::error::Error; 12 | 13 | use crate::hls::HlsDownloader; 14 | use hls::download_to_file; 15 | use reqwest::{Client, Request}; 16 | 17 | pub fn download_hls( 18 | http: Client, 19 | request: Request, 20 | filter: Option bool>, 21 | ) -> DownloadStream { 22 | HlsDownloader::new(request, http, filter).download() 23 | } 24 | 25 | pub fn download_hls_named( 26 | http: Client, 27 | request: Request, 28 | name: String, 29 | filter: Option bool>, 30 | ) -> DownloadStream { 31 | HlsDownloader::new_named(request, http, name, filter).download() 32 | } 33 | 34 | pub fn download_hls_master_first( 35 | http: Client, 36 | request: Request, 37 | filter: Option bool>, 38 | ) -> DownloadStream { 39 | HlsDownloader::new_master_first(request, http, filter).download() 40 | } 41 | 42 | pub fn download_chunked(http: Client, request: Request) -> DownloadStream { 43 | let (dl, tx) = DownloadStream::new(); 44 | 45 | tokio::spawn(download_to_file( 46 | http, 47 | request, 48 | tx, 49 | None, 50 | Some(Duration::from_secs(60)), 51 | )); 52 | dl 53 | } 54 | -------------------------------------------------------------------------------- /rsget_lib/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::new_ret_no_self)] 2 | #![deny(rust_2018_idioms)] 3 | 4 | #[macro_use] 5 | extern crate serde_derive; 6 | 7 | use crate::utils::error::StreamResult; 8 | 9 | use std::boxed::Box; 10 | 11 | use stream_lib::DownloadStream; 12 | 13 | use async_trait::async_trait; 14 | 15 | /// Status of the live stream 16 | pub enum Status { 17 | /// Stream is online. 18 | Online, 19 | /// Stream is offline. 20 | Offline, 21 | /// The status of the stream could not be determined. 22 | Unknown, 23 | } 24 | 25 | #[async_trait] 26 | pub trait Streamable { 27 | /// Creates a new streamable 28 | async fn new(url: String) -> StreamResult> 29 | where 30 | Self: Sized + Sync; 31 | /// Returns the title of the stream if possible 32 | async fn get_title(&self) -> StreamResult; 33 | /// Returns the author of the stream if possible 34 | async fn get_author(&self) -> StreamResult; 35 | /// Returns if the stream is online 36 | async fn is_online(&self) -> StreamResult; 37 | /// Gets the url of the stream 38 | async fn get_stream(&self) -> StreamResult; 39 | /// Returns what extension the stream should be 40 | async fn get_ext(&self) -> StreamResult; 41 | /// Gets the default name of the stream 42 | async fn get_default_name(&self) -> StreamResult; 43 | } 44 | 45 | // impl Streamable for Box 46 | // where S: Streamable 47 | // { } 48 | 49 | pub mod plugins; 50 | pub mod utils; 51 | -------------------------------------------------------------------------------- /stream_lib/src/error.rs: -------------------------------------------------------------------------------- 1 | use hls_m3u8::Error as HlsError; 2 | use reqwest::Error as ReqwestError; 3 | use std::{ 4 | error::Error as StdError, 5 | fmt::{Display, Formatter, Result as FmtResult}, 6 | }; 7 | use tokio::io::Error as TokioIoError; 8 | use url::ParseError; 9 | 10 | #[derive(Debug)] 11 | pub enum Error { 12 | /// M3U8 error 13 | Hls(HlsError), 14 | /// Http error. 15 | Reqwest(ReqwestError), 16 | /// Url error. 17 | Url(ParseError), 18 | /// Tokio IO error 19 | TIO(TokioIoError), 20 | } 21 | 22 | impl From for Error { 23 | fn from(err: HlsError) -> Self { 24 | Error::Hls(err) 25 | } 26 | } 27 | 28 | impl From for Error { 29 | fn from(err: ReqwestError) -> Self { 30 | Error::Reqwest(err) 31 | } 32 | } 33 | 34 | impl From for Error { 35 | fn from(err: ParseError) -> Self { 36 | Error::Url(err) 37 | } 38 | } 39 | 40 | impl From for Error { 41 | fn from(err: TokioIoError) -> Self { 42 | Error::TIO(err) 43 | } 44 | } 45 | 46 | impl Display for Error { 47 | fn fmt(&self, f: &mut Formatter) -> FmtResult { 48 | match self { 49 | Error::Hls(hls) => { 50 | f.write_str("Hls Error: ")?; 51 | Display::fmt(hls, f) 52 | } 53 | Error::Reqwest(req) => { 54 | f.write_str("Reqwest Error: ")?; 55 | Display::fmt(req, f) 56 | } 57 | Error::Url(url) => { 58 | f.write_str("Url Error: ")?; 59 | Display::fmt(url, f) 60 | } 61 | Error::TIO(io) => { 62 | f.write_str("Tokio IO Error: ")?; 63 | Display::fmt(io, f) 64 | } 65 | } 66 | } 67 | } 68 | 69 | impl StdError for Error {} 70 | -------------------------------------------------------------------------------- /rsget_lib/src/utils/downloaders.rs: -------------------------------------------------------------------------------- 1 | use crate::utils::error::RsgetError; 2 | use crate::utils::error::StreamError; 3 | 4 | use std::process::Command; 5 | 6 | use serde::de::DeserializeOwned; 7 | 8 | use reqwest; 9 | use reqwest::Client as RClient; 10 | 11 | #[derive(Debug, Clone)] 12 | pub struct DownloadClient { 13 | pub rclient: RClient, 14 | } 15 | 16 | pub fn ffmpeg_download(url: String, path: String) -> Result<(), StreamError> { 17 | let comm = Command::new("ffmpeg") 18 | .arg("-i") 19 | .arg(url) 20 | .arg("-c") 21 | .arg("copy") 22 | .arg(path) 23 | .status() 24 | .expect("ffmpeg failed to start"); 25 | match comm.code() { 26 | Some(c) => { 27 | info!("Ffmpeg returned: {}", c); 28 | Ok(()) 29 | } 30 | None => { 31 | info!("Err: Ffmpeg failed"); 32 | Err(StreamError::Rsget(RsgetError::new("Ffmpeg failed"))) 33 | } 34 | } 35 | } 36 | 37 | impl DownloadClient { 38 | pub fn new() -> Result { 39 | Ok(DownloadClient { 40 | rclient: RClient::new(), 41 | }) 42 | } 43 | 44 | pub fn download_to_string(&self, req: reqwest::Request) -> Result { 45 | let c = &self.rclient; 46 | let mut res = c.execute(req)?; 47 | res.text().map_err(StreamError::from) 48 | } 49 | 50 | pub fn download_and_de( 51 | &self, 52 | req: reqwest::Request, 53 | ) -> Result { 54 | let c = &self.rclient; 55 | let mut res = c.execute(req)?; 56 | let json: T = res.json()?; 57 | Ok(json) 58 | } 59 | 60 | pub fn make_request( 61 | &self, 62 | uri: &str, 63 | headers: Option<(&str, &str)>, 64 | ) -> Result { 65 | let c = &self.rclient; 66 | match headers { 67 | Some(a) => c 68 | .get(uri) 69 | .header(a.0, a.1) 70 | .build() 71 | .map_err(StreamError::from), 72 | None => c.get(uri).build().map_err(StreamError::from), 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /rsget_lib/src/utils/sites.rs: -------------------------------------------------------------------------------- 1 | /* 2 | use crate::plugins::{ 3 | afreeca::Afreeca, dlive::DLive, douyin::Douyin, douyu::Douyu, huya::Huya, inke::Inke, 4 | mixer::Mixer, tiktok::TikTok, twitch::Twitch, vlive::Vlive, 5 | }; 6 | */ 7 | //use crate::plugins::{Afreeca, Bilibili, DLive, Twitch, Vlive}; 8 | use crate::plugins::{Afreeca, Bilibili, DLive, Drdk, Twitch, Vlive}; 9 | use crate::utils::error::RsgetError; 10 | use crate::utils::error::StreamError; 11 | use crate::utils::error::StreamResult; 12 | use crate::Streamable; 13 | use regex::Regex; 14 | 15 | pub async fn get_site(input: &str) -> StreamResult> { 16 | match _get_site(input).await { 17 | Ok(s) => Ok(s), 18 | Err(StreamError::Rsget(_)) => { 19 | let res = reqwest::get(input).await?; 20 | let final_url = res.url().as_str(); 21 | _get_site(final_url).await 22 | } 23 | Err(why) => Err(why), 24 | } 25 | } 26 | 27 | async fn _get_site(input: &str) -> StreamResult> { 28 | let re_drdk: Regex = Regex::new(r"^(?:https?://)?(?:www\.)?dr\.dk/drtv/kanal/[a-zA-Z0-9-_]+")?; 29 | let re_afreeca: Regex = Regex::new( 30 | r"^(?:https?://)?(?:www\.)?(?:play\.)?afreecatv.com/[a-zA-Z0-9]+/?(?:/[0-9]+)?", 31 | )?; 32 | let re_dlive = Regex::new(r"^(?:https?://)?(?:www\.)?dlive\.tv/[a-zA-Z0-9]+\??.*")?; 33 | let re_twitch = Regex::new(r"^(?:https?://)?(?:www\.)?twitch\.tv/([a-zA-Z0-9_]+)")?; 34 | let re_bilibili = Regex::new(r"^(?:https?://)?(?:www\.)?live\.bilibili\.com/([0-9]+)")?; 35 | //let re_douyu: Regex = Regex::new(r"^(?:https?://)?(?:www\.)?douyu\.com/[a-zA-Z0-9]+/?")?; 36 | /* 37 | let re_inke: Regex = Regex::new(r"^(?:https?://)?(?:www\.)?inke\.cn/live\.html\?uid=[0-9]+")?; 38 | let re_douyin: Regex = Regex::new(r"^(?:https?://)?(?:www\.)?iesdouyin\.com/.*")?; 39 | let re_tiktok: Regex = 40 | Regex::new(r"^(?:https?://)?(?:www\.)?(?:m\.)?tiktok\.com/v/(?:[a-zA-Z0-9]+)(?:\.html)?")?; 41 | let re_huya: Regex = Regex::new(r"^(?:https?://)?(?:www\.)?huya\.com/[a-zA-Z0-9]+")?;*/ 42 | let re_vlive: Regex = Regex::new(r"^(?:https?://)?(?:www\.)?vlive\.tv/video/(\d+)")?; 43 | match input { 44 | //url if re_douyu.is_match(url) => Ok(Douyu::new(String::from(url))?), 45 | url if re_afreeca.is_match(url) => Ok(Afreeca::new(String::from(url)).await?), 46 | url if re_bilibili.is_match(url) => Ok(Bilibili::new(String::from(url)).await?), 47 | url if re_dlive.is_match(url) => Ok(DLive::new(String::from(url)).await?), 48 | url if re_drdk.is_match(url) => Ok(Drdk::new(String::from(url)).await?), 49 | url if re_twitch.is_match(url) => Ok(Twitch::new(String::from(url)).await?), 50 | url if re_vlive.is_match(url) => Ok(Vlive::new(String::from(url)).await?), 51 | /* 52 | url if re_inke.is_match(url) => Ok(Inke::new(String::from(url))?), 53 | url if re_douyin.is_match(url) => Ok(Douyin::new(String::from(url)).await?), 54 | url if re_tiktok.is_match(url) => Ok(TikTok::new(String::from(url))?), 55 | url if re_huya.is_match(url) => Ok(Huya::new(String::from(url))?), 56 | url if re_twitch.is_match(url) => Ok(Twitch::new(String::from(url))?), 57 | url if re_dlive.is_match(url) => Ok(DLive::new(String::from(url))?), 58 | 59 | */ 60 | _ => Err(StreamError::Rsget(RsgetError::new("Site not supported."))), 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /rsget_lib/src/plugins/drdk.rs: -------------------------------------------------------------------------------- 1 | // https://www.dr.dk/drtv/kanal/ 2 | 3 | use std::collections::BTreeMap; 4 | 5 | use async_trait::async_trait; 6 | use chrono::{DateTime, Datelike, Local, Timelike}; 7 | use regex::Regex; 8 | use stream_lib::DownloadStream; 9 | 10 | use crate::{ 11 | utils::error::{RsgetError, StreamError, StreamResult}, 12 | Status, Streamable, 13 | }; 14 | 15 | pub struct Drdk { 16 | hls_url: String, 17 | title: String, 18 | } 19 | 20 | #[async_trait] 21 | impl Streamable for Drdk { 22 | async fn new(url: String) -> StreamResult> 23 | where 24 | Self: Sized + Sync, 25 | { 26 | let re_drdk = Regex::new(r"^(?:https?://)?(?:www\.)?dr\.dk/drtv/kanal/[a-zA-Z0-9-_]+$")?; 27 | if !re_drdk.is_match(&url) { 28 | return Err(StreamError::Rsget(RsgetError::new("unsupported url"))); 29 | } 30 | 31 | let http = reqwest::Client::new(); 32 | 33 | let resp = http.get(&url).send().await?; 34 | let html = resp.text().await?; 35 | 36 | let window_data_re = Regex::new(r"")?; 37 | 38 | let json = &window_data_re 39 | .captures(&html) 40 | .ok_or(StreamError::Rsget(RsgetError::new( 41 | "Could not find window data", 42 | )))?[1]; 43 | 44 | let mut stream = serde_json::Deserializer::from_str(json.trim()).into_iter::(); 45 | 46 | let mut window_data = stream 47 | .next() 48 | .ok_or(StreamError::Rsget(RsgetError::new("could not find json")))??; 49 | let (_id, detail) = window_data 50 | .cache 51 | .item_detail 52 | .pop_first() 53 | .ok_or(StreamError::Rsget(RsgetError::new( 54 | "Could not item details", 55 | )))?; 56 | let hls_url = detail.item.custom_fields.hls_url; 57 | let title = detail.item.title; 58 | 59 | Ok(Box::new(Drdk { hls_url, title })) 60 | } 61 | async fn get_title(&self) -> StreamResult { 62 | Ok(self.title.clone()) 63 | } 64 | async fn get_author(&self) -> StreamResult { 65 | Ok("DR.DK".to_owned()) 66 | } 67 | async fn is_online(&self) -> StreamResult { 68 | Ok(Status::Unknown) 69 | } 70 | async fn get_stream(&self) -> StreamResult { 71 | let http = reqwest::Client::new(); 72 | let request = http.get(&self.hls_url).build()?; 73 | Ok(stream_lib::download_hls_master_first(http, request, None)) 74 | } 75 | async fn get_ext(&self) -> StreamResult { 76 | Ok("ts".to_owned()) 77 | } 78 | async fn get_default_name(&self) -> StreamResult { 79 | let local: DateTime = Local::now(); 80 | Ok(format!( 81 | "DRTV-{:04}-{:02}-{:02}-{:02}-{:02}-{}.{}", 82 | local.year(), 83 | local.month(), 84 | local.day(), 85 | local.hour(), 86 | local.minute(), 87 | &self.title, 88 | self.get_ext().await? 89 | )) 90 | } 91 | } 92 | 93 | #[derive(Debug, Clone, Serialize, Deserialize)] 94 | struct WindowData { 95 | cache: Cache, 96 | } 97 | 98 | #[derive(Debug, Clone, Serialize, Deserialize)] 99 | struct Cache { 100 | #[serde(rename = "itemDetail")] 101 | item_detail: BTreeMap, 102 | } 103 | 104 | #[derive(Debug, Clone, Serialize, Deserialize)] 105 | struct Item { 106 | item: ItemDetail, 107 | } 108 | 109 | #[derive(Debug, Clone, Serialize, Deserialize)] 110 | struct ItemDetail { 111 | #[serde(rename = "customFields")] 112 | custom_fields: CustomFields, 113 | title: String, 114 | } 115 | 116 | #[derive(Debug, Clone, Serialize, Deserialize)] 117 | struct CustomFields { 118 | #[serde(rename = "hlsURL")] 119 | hls_url: String, 120 | } 121 | -------------------------------------------------------------------------------- /rsget_lib/src/plugins/dlive.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused)] 2 | 3 | use crate::{Status, Streamable}; 4 | use regex::Regex; 5 | use stream_lib::DownloadStream; 6 | use tracing::debug; 7 | 8 | use crate::utils::error::RsgetError; 9 | use crate::utils::error::StreamError; 10 | use crate::utils::error::StreamResult; 11 | 12 | use chrono::prelude::*; 13 | 14 | use async_trait::async_trait; 15 | use serde_json::Value; 16 | 17 | #[derive(Debug, Clone)] 18 | pub struct DLive { 19 | client: reqwest::Client, 20 | url: String, 21 | apollo_state: Value, 22 | } 23 | 24 | #[async_trait] 25 | impl Streamable for DLive { 26 | async fn new(url: String) -> StreamResult> { 27 | let client = reqwest::Client::new(); 28 | 29 | let room_id_re = Regex::new(r"^(?:https?://)?(?:www\.)?dlive\.tv/([a-zA-Z0-9]+)")?; 30 | let cap = match room_id_re.captures(&url) { 31 | Some(capture) => capture, 32 | None => return Err(StreamError::Rsget(RsgetError::new("No capture found"))), 33 | }; 34 | let site_url = format!("https://dlive.tv/{}", &cap[1]); 35 | let res = client.get(&site_url).send().await?.text().await?; 36 | 37 | let apollo_state_re = Regex::new(r"__APOLLO_STATE__=(.*);\(function\(\)")?; 38 | let apollo_state_cap = apollo_state_re.captures(&res).ok_or_else(|| { 39 | StreamError::Rsget(RsgetError::new("Regex did not find any hostinfo")) 40 | })?; 41 | let apollo_state: Value = match serde_json::from_str(&apollo_state_cap[1]) { 42 | Ok(state) => state, 43 | Err(why) => return Err(StreamError::Json(why)), 44 | }; 45 | 46 | let aps = apollo_state["defaultClient"] 47 | .as_object() 48 | .ok_or(RsgetError::Offline)? 49 | .into_iter() 50 | .find(|e| e.0.starts_with("user:")) 51 | .ok_or(RsgetError::Offline)? 52 | .1 53 | .clone(); 54 | 55 | let xy = DLive { 56 | client, 57 | url: url.clone(), 58 | apollo_state: aps, 59 | }; 60 | debug!("{:#?}", &xy); 61 | Ok(Box::new(xy)) 62 | } 63 | 64 | async fn get_title(&self) -> StreamResult { 65 | Ok("".to_string()) 66 | } 67 | 68 | async fn get_author(&self) -> StreamResult { 69 | Ok(self.apollo_state["displayname"] 70 | .as_str() 71 | .unwrap() 72 | .trim_end_matches('"') 73 | .to_string()) 74 | } 75 | 76 | async fn is_online(&self) -> StreamResult { 77 | if !self.apollo_state["livestream"].is_null() { 78 | Ok(Status::Online) 79 | } else { 80 | Ok(Status::Unknown) 81 | } 82 | } 83 | 84 | async fn get_stream(&self) -> StreamResult { 85 | let url = format!( 86 | "https://live.prd.dlive.tv/hls/live/{}.m3u8", 87 | &self.apollo_state["username"] 88 | .as_str() 89 | .unwrap() 90 | .trim_start_matches("%22") 91 | .trim_end_matches("%22") 92 | ); 93 | Ok(stream_lib::download_hls_named( 94 | self.client.clone(), 95 | self.client.get(url).build()?, 96 | String::from("src"), 97 | None, 98 | )) 99 | } 100 | 101 | async fn get_ext(&self) -> StreamResult { 102 | Ok(String::from("mp4")) 103 | } 104 | 105 | async fn get_default_name(&self) -> StreamResult { 106 | let local: DateTime = Local::now(); 107 | Ok(format!( 108 | "{}-{:04}-{:02}-{:02}-{:02}-{:02}.{}", 109 | self.get_author().await?, 110 | local.year(), 111 | local.month(), 112 | local.day(), 113 | local.hour(), 114 | local.minute(), 115 | self.get_ext().await? 116 | )) 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /rsget_lib/src/plugins/douyin.rs: -------------------------------------------------------------------------------- 1 | use crate::Streamable; 2 | use regex::Regex; 3 | 4 | use crate::utils::error::RsgetError; 5 | use crate::utils::error::StreamError; 6 | 7 | use crate::utils::downloaders::DownloadClient; 8 | 9 | use stream_lib::StreamType; 10 | 11 | /* 12 | #[derive(Serialize, Deserialize, Clone, Debug)] 13 | pub struct DouyinRoot { 14 | #[serde(rename = "hasData")] 15 | pub has_data: i64, 16 | #[serde(rename = "videoWidth")] 17 | pub video_width: i64, 18 | #[serde(rename = "videoHeight")] 19 | pub video_height: i64, 20 | #[serde(rename = "playAddr")] 21 | pub play_addr: String, 22 | pub cover: String, 23 | } 24 | */ 25 | 26 | #[derive(Clone, Debug)] 27 | pub struct Douyin { 28 | pub url: String, 29 | pub video_id: String, 30 | pub douyin_url: String, //DouyinRoot, 31 | pub description: String, 32 | pub author: String, 33 | client: DownloadClient, 34 | } 35 | 36 | impl Streamable for Douyin { 37 | fn new(url: String) -> Result, StreamError> { 38 | let dc = DownloadClient::new()?; 39 | let site_req = dc.make_request(&url, None)?; 40 | let res: Result = dc.download_to_string(site_req); 41 | match res { 42 | Ok(some) => { 43 | let url_re: Regex = Regex::new( 44 | r"^(?:https?://)?(?:www\.)?iesdouyin\.com/share/video/([a-zA-Z0-9]+)/.*", 45 | )?; 46 | let video_re = Regex::new(r#"playAddr:\s*"(.+)""#)?; 47 | let description_re = Regex::new(r#"

([^<]*)

"#)?; 48 | let author_re = Regex::new(r#"

@([^<]*)

"#)?; 49 | let id_cap = url_re.captures(&url).ok_or_else(|| { 50 | StreamError::Rsget(RsgetError::new("Regex did not find any video id")) 51 | })?; 52 | let video_cap = video_re.captures(&some).ok_or_else(|| { 53 | StreamError::Rsget(RsgetError::new("Regex did not find any hostinfo")) 54 | })?; 55 | let description_cap = description_re.captures(&some).ok_or_else(|| { 56 | StreamError::Rsget(RsgetError::new("Regex did not find any description")) 57 | })?; 58 | let author_cap = author_re.captures(&some).ok_or_else(|| { 59 | StreamError::Rsget(RsgetError::new("Regex did not find any author")) 60 | })?; 61 | 62 | let ret_val = Douyin { 63 | client: dc, 64 | url: url.clone(), 65 | video_id: String::from(&id_cap[1]), 66 | douyin_url: String::from(&video_cap[1]), 67 | description: String::from(&description_cap[1]), 68 | author: String::from(&author_cap[1]), 69 | }; 70 | info!("{:#?}", &ret_val); 71 | Ok(Box::new(ret_val)) 72 | } 73 | Err(why) => Err(why), 74 | } 75 | } 76 | 77 | fn get_title(&self) -> Option { 78 | Some(self.description.clone()) 79 | } 80 | 81 | fn get_author(&self) -> Option { 82 | Some(self.author.clone()) 83 | } 84 | 85 | fn is_online(&self) -> bool { 86 | true 87 | } 88 | 89 | fn get_stream(&self) -> Result { 90 | Ok(StreamType::Chuncked( 91 | self.client.rclient.get(&self.douyin_url).build()?, 92 | )) 93 | } 94 | 95 | fn get_ext(&self) -> String { 96 | String::from("mp4") 97 | } 98 | 99 | fn get_default_name(&self) -> String { 100 | format!( 101 | "{}-{}-{}.{}", 102 | self.video_id, 103 | self.get_title().unwrap(), 104 | self.get_author().unwrap(), 105 | self.get_ext() 106 | ) 107 | } 108 | fn get_reqwest_client(&self) -> &reqwest::Client { 109 | &self.client.rclient 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /rsget_lib/src/plugins/bilibili.rs: -------------------------------------------------------------------------------- 1 | use crate::{Status, Streamable}; 2 | use regex::Regex; 3 | use stream_lib::DownloadStream; 4 | 5 | use crate::utils::error::RsgetError; 6 | use crate::utils::error::StreamError; 7 | use crate::utils::error::StreamResult; 8 | 9 | use chrono::prelude::*; 10 | 11 | use async_trait::async_trait; 12 | 13 | const USER_AGENT: &str = "Mozilla/5.0 (X11; FreeBSD amd64; rv:78.0) Gecko/20100101 Firefox/78.0"; 14 | 15 | #[derive(Clone, Debug, Serialize, Deserialize)] 16 | struct RoomInitHead { 17 | data: RoomInit, 18 | } 19 | 20 | #[derive(Clone, Debug, Serialize, Deserialize)] 21 | struct RoomInit { 22 | room_id: u64, 23 | live_status: u64, 24 | } 25 | 26 | #[derive(Clone, Debug, Serialize, Deserialize)] 27 | struct PlayUrlHead { 28 | data: PlayUrl, 29 | } 30 | 31 | #[derive(Clone, Debug, Serialize, Deserialize)] 32 | struct PlayUrl { 33 | durl: Vec, 34 | } 35 | 36 | #[derive(Clone, Debug, Serialize, Deserialize)] 37 | struct Durl { 38 | url: String, 39 | } 40 | 41 | #[derive(Debug, Clone)] 42 | pub struct Bilibili { 43 | client: reqwest::Client, 44 | room_id: String, 45 | room_init: RoomInit, 46 | durl_list: Vec, 47 | } 48 | 49 | #[async_trait] 50 | impl Streamable for Bilibili { 51 | async fn new(url: String) -> StreamResult> { 52 | let room_id_re = Regex::new(r"^(?:https?://)?(?:www\.)?live\.bilibili\.com/([0-9]+)")?; 53 | 54 | let cap = match room_id_re.captures(&url) { 55 | Some(capture) => capture, 56 | None => return Err(StreamError::Rsget(RsgetError::new("No room_id found"))), 57 | }; 58 | let room_init_url = format!( 59 | "https://api.live.bilibili.com/room/v1/Room/room_init?id={}", 60 | &cap[1] 61 | ); 62 | 63 | let room_id = String::from(&cap[1]); 64 | 65 | let client = reqwest::Client::new(); 66 | 67 | let room_init = client 68 | .get(&room_init_url) 69 | .send() 70 | .await? 71 | .json::() 72 | .await? 73 | .data; 74 | 75 | if room_init.live_status != 1 { 76 | return Err(RsgetError::Offline.into()); 77 | } 78 | 79 | let durls = client 80 | .get("https://api.live.bilibili.com/room/v1/Room/playUrl") 81 | .query(&[("cid", &cap[1]), ("quality", "0"), ("platform", "web")]) 82 | .header("User-Agent", USER_AGENT) 83 | .header("Accept", "*/*") 84 | .header("Accept-Language", "en-US,en;q=0.5") 85 | .send() 86 | .await? 87 | .json::() 88 | .await? 89 | .data 90 | .durl; 91 | 92 | Ok(Box::new(Bilibili { 93 | client, 94 | room_id, 95 | room_init, 96 | durl_list: durls, 97 | })) 98 | } 99 | 100 | async fn get_title(&self) -> StreamResult { 101 | Ok("".to_string()) 102 | } 103 | 104 | async fn get_author(&self) -> StreamResult { 105 | Ok(self.room_id.clone()) 106 | } 107 | 108 | async fn is_online(&self) -> StreamResult { 109 | if self.room_init.live_status == 1 { 110 | Ok(Status::Online) 111 | } else { 112 | Ok(Status::Offline) 113 | } 114 | } 115 | 116 | async fn get_stream(&self) -> StreamResult { 117 | Ok(stream_lib::download_chunked( 118 | self.client.clone(), 119 | self.client 120 | .get(&self.durl_list[0].url) 121 | .header("User-Agent", USER_AGENT) 122 | .build()?, 123 | )) 124 | } 125 | 126 | async fn get_ext(&self) -> StreamResult { 127 | Ok(String::from("flv")) 128 | } 129 | 130 | async fn get_default_name(&self) -> StreamResult { 131 | let local: DateTime = Local::now(); 132 | Ok(format!( 133 | "{}-{:04}-{:02}-{:02}-{:02}-{:02}.{}", 134 | self.get_author().await?, 135 | local.year(), 136 | local.month(), 137 | local.day(), 138 | local.hour(), 139 | local.minute(), 140 | self.get_ext().await? 141 | )) 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /rsget_lib/src/plugins/inke.rs: -------------------------------------------------------------------------------- 1 | use crate::Streamable; 2 | use regex::Regex; 3 | 4 | use crate::utils::error::StreamError; 5 | 6 | use chrono::prelude::*; 7 | 8 | use stream_lib::StreamType; 9 | 10 | use crate::utils::downloaders::DownloadClient; 11 | 12 | #[allow(dead_code)] 13 | #[allow(non_snake_case)] 14 | #[derive(Clone, Debug, Deserialize)] 15 | pub struct InkeUser { 16 | uid: usize, 17 | nick: String, 18 | gender: usize, 19 | city: String, 20 | level: usize, 21 | pic: String, 22 | isfollow: usize, 23 | } 24 | 25 | #[allow(dead_code)] 26 | #[allow(non_snake_case)] 27 | #[derive(Clone, Debug, Deserialize)] 28 | pub struct InkeAddr { 29 | liveid: String, 30 | stream_addr: String, 31 | hls_stream_addr: String, 32 | rtmp_stream_addr: String, 33 | } 34 | 35 | #[allow(dead_code)] 36 | #[allow(non_snake_case)] 37 | #[derive(Clone, Debug, Deserialize)] 38 | pub struct InkeLiveInfo { 39 | slot: usize, 40 | user: InkeUser, 41 | online_users: usize, 42 | name: String, 43 | city: String, 44 | pub_stat: usize, 45 | landscape: usize, 46 | rotate: usize, 47 | live_type: String, 48 | cover_img: String, 49 | image: String, 50 | points: usize, 51 | liveid: usize, 52 | status: usize, 53 | } 54 | 55 | #[allow(dead_code)] 56 | #[allow(non_snake_case)] 57 | #[derive(Clone, Debug, Deserialize)] 58 | pub struct InkeData { 59 | live_info: InkeLiveInfo, 60 | room_type: String, 61 | public_live_info: Option, 62 | live_addr: Vec, 63 | lived_addr: Vec, 64 | } 65 | 66 | #[allow(dead_code)] 67 | #[allow(non_snake_case)] 68 | #[derive(Clone, Debug, Deserialize)] 69 | pub struct InkeStruct { 70 | error_code: usize, 71 | message: String, 72 | data: InkeData, 73 | } 74 | 75 | #[derive(Clone, Debug)] 76 | pub struct Inke { 77 | pub url: String, 78 | pub room_id: String, 79 | pub inke_info: InkeStruct, 80 | client: DownloadClient, 81 | } 82 | 83 | impl Streamable for Inke { 84 | fn new(url: String) -> Result, StreamError> { 85 | let dc = DownloadClient::new()?; 86 | let re_inke: Regex = 87 | Regex::new(r"^(?:https?://)?(?:www\.)?inke\.cn/live\.html\?uid=([0-9]+)").unwrap(); 88 | let cap = re_inke.captures(&url).unwrap(); 89 | let json_url = format!("http://baseapi.busi.inke.cn/live/LiveInfo?uid={}", &cap[1]); 90 | let json_req = dc.make_request(&json_url, None)?; 91 | let jres = dc.download_and_de::(json_req); 92 | match jres { 93 | Ok(jre) => { 94 | let ik = Inke { 95 | url: String::from(url.as_str()), 96 | room_id: String::from(&cap[1]), 97 | inke_info: jre, 98 | client: dc, 99 | }; 100 | debug!("{:#?}", ik); 101 | Ok(Box::new(ik)) 102 | } 103 | Err(why) => Err(why), 104 | } 105 | } 106 | 107 | fn get_title(&self) -> Option { 108 | Some(self.inke_info.data.live_info.name.clone()) 109 | } 110 | 111 | fn get_author(&self) -> Option { 112 | Some(self.inke_info.data.live_info.user.nick.clone()) 113 | } 114 | 115 | fn is_online(&self) -> bool { 116 | self.inke_info.error_code == 0 117 | } 118 | 119 | fn get_stream(&self) -> Result { 120 | Ok(StreamType::Chuncked( 121 | self.client 122 | .rclient 123 | .get(&self.inke_info.data.live_addr[0].stream_addr) 124 | .build()?, 125 | )) 126 | } 127 | 128 | fn get_ext(&self) -> String { 129 | String::from("flv") 130 | } 131 | 132 | fn get_default_name(&self) -> String { 133 | let local: DateTime = Local::now(); 134 | format!( 135 | "{}-{:04}-{:02}-{:02}-{:02}-{:02}-{}-{}.{}", 136 | self.room_id, 137 | local.year(), 138 | local.month(), 139 | local.day(), 140 | local.hour(), 141 | local.minute(), 142 | self.get_author().unwrap(), 143 | self.get_title().unwrap(), 144 | self.get_ext() 145 | ) 146 | } 147 | fn get_reqwest_client(&self) -> &reqwest::Client { 148 | &self.client.rclient 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /rsget_lib/src/utils/error.rs: -------------------------------------------------------------------------------- 1 | use std::string::FromUtf8Error; 2 | 3 | use serde_json::Error as JsonError; 4 | use std::{ 5 | error::Error as StdError, 6 | fmt::{Display, Error as FmtError, Formatter, Result as FmtResult}, 7 | io::Error as IoError, 8 | }; 9 | 10 | use http::header::ToStrError; 11 | use http::uri::InvalidUri; 12 | 13 | use http::Error as HttpError; 14 | 15 | use hls_m3u8::Error as HlsError; 16 | 17 | use serde_urlencoded::ser::Error as UrlEncError; 18 | 19 | use reqwest::Error as ReqwestError; 20 | 21 | use regex::Error as RegexError; 22 | 23 | use stream_lib::Error as StreamLibError; 24 | 25 | pub type StreamResult = Result; 26 | 27 | #[derive(Debug)] 28 | pub enum RsgetError { 29 | Offline, 30 | Other(String), 31 | } 32 | 33 | impl RsgetError { 34 | pub fn new(msg: &str) -> RsgetError { 35 | RsgetError::Other(String::from(msg)) 36 | } 37 | } 38 | 39 | impl Display for RsgetError { 40 | fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { 41 | let printable = match self { 42 | RsgetError::Offline => "Stream offline", 43 | RsgetError::Other(s) => s, 44 | }; 45 | write!(f, "{}", printable) 46 | } 47 | } 48 | 49 | impl StdError for RsgetError { 50 | fn description(&self) -> &str { 51 | match self { 52 | RsgetError::Offline => "Stream offline", 53 | RsgetError::Other(s) => s, 54 | } 55 | } 56 | } 57 | 58 | #[derive(Debug)] 59 | pub enum StreamError { 60 | /// An error that occurred while formatting a string. 61 | Fmt(FmtError), 62 | /// An error from the `serde_json` crate while deserializing the body of an 63 | /// HTTP response. 64 | Json(JsonError), 65 | /// An error from the `reqwest` crate while performing an HTTP request. 66 | /// RsgetError 67 | Rsget(RsgetError), 68 | /// IO-Error 69 | Io(IoError), 70 | /// UriError 71 | Uri(InvalidUri), 72 | /// ToStrError 73 | ToStr(ToStrError), 74 | /// HTTP Error 75 | Http(HttpError), 76 | /// HLS Error 77 | Hls(HlsError), 78 | /// FromUtf8error 79 | Utf8(FromUtf8Error), 80 | /// Serde URL Encode error 81 | UrlEnc(UrlEncError), 82 | /// Reqwest Error 83 | Reqwest(ReqwestError), 84 | /// Regex Error 85 | Regex(RegexError), 86 | /// Stream lib 87 | Stream(StreamLibError), 88 | } 89 | 90 | impl Display for StreamError { 91 | fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { 92 | let error_str = match *self { 93 | StreamError::Fmt(ref inner) => inner.to_string(), 94 | StreamError::Json(ref inner) => inner.to_string(), 95 | StreamError::Rsget(ref inner) => inner.to_string(), 96 | StreamError::Io(ref inner) => inner.to_string(), 97 | StreamError::Uri(ref inner) => inner.to_string(), 98 | StreamError::ToStr(ref inner) => inner.to_string(), 99 | StreamError::Http(ref inner) => inner.to_string(), 100 | StreamError::Hls(ref inner) => inner.to_string(), 101 | StreamError::Utf8(ref inner) => inner.to_string(), 102 | StreamError::UrlEnc(ref inner) => inner.to_string(), 103 | StreamError::Reqwest(ref inner) => inner.to_string(), 104 | StreamError::Regex(ref inner) => inner.to_string(), 105 | StreamError::Stream(ref inner) => inner.to_string(), 106 | }; 107 | f.write_str(&error_str) 108 | } 109 | } 110 | 111 | impl From for StreamError { 112 | fn from(err: FmtError) -> Self { 113 | StreamError::Fmt(err) 114 | } 115 | } 116 | 117 | impl From for StreamError { 118 | fn from(err: JsonError) -> Self { 119 | StreamError::Json(err) 120 | } 121 | } 122 | 123 | impl From for StreamError { 124 | fn from(err: IoError) -> Self { 125 | StreamError::Io(err) 126 | } 127 | } 128 | 129 | impl From for StreamError { 130 | fn from(err: InvalidUri) -> Self { 131 | StreamError::Uri(err) 132 | } 133 | } 134 | 135 | impl From for StreamError { 136 | fn from(err: ToStrError) -> Self { 137 | StreamError::ToStr(err) 138 | } 139 | } 140 | 141 | impl From for StreamError { 142 | fn from(err: HttpError) -> Self { 143 | StreamError::Http(err) 144 | } 145 | } 146 | 147 | impl From for StreamError { 148 | fn from(err: HlsError) -> Self { 149 | StreamError::Hls(err) 150 | } 151 | } 152 | 153 | impl From for StreamError { 154 | fn from(err: FromUtf8Error) -> Self { 155 | StreamError::Utf8(err) 156 | } 157 | } 158 | 159 | impl From for StreamError { 160 | fn from(err: UrlEncError) -> Self { 161 | StreamError::UrlEnc(err) 162 | } 163 | } 164 | 165 | impl From for StreamError { 166 | fn from(err: ReqwestError) -> Self { 167 | StreamError::Reqwest(err) 168 | } 169 | } 170 | 171 | impl From for StreamError { 172 | fn from(err: RegexError) -> Self { 173 | StreamError::Regex(err) 174 | } 175 | } 176 | 177 | impl From for StreamError { 178 | fn from(err: RsgetError) -> Self { 179 | StreamError::Rsget(err) 180 | } 181 | } 182 | 183 | impl From for StreamError { 184 | fn from(err: StreamLibError) -> Self { 185 | StreamError::Stream(err) 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /stream_lib/src/hls/watch.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use hls_m3u8::MediaPlaylist; 4 | use patricia_tree::PatriciaSet; 5 | 6 | use reqwest::{Client, Request, Url}; 7 | use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; 8 | use tracing::{debug, trace, warn}; 9 | 10 | use crate::{ 11 | hls::{clone_request, HlsQueue, HLS_MAX_RETRIES}, 12 | Error, 13 | }; 14 | 15 | pub struct HlsWatch { 16 | tx: UnboundedSender, 17 | request: Request, 18 | http: Client, 19 | links: PatriciaSet, 20 | master_url: Url, 21 | timeout: Duration, 22 | fail_counter: usize, 23 | filter: Option bool>, 24 | } 25 | 26 | impl HlsWatch { 27 | /// Filter will filter any url that returns `false`, if `None` it will not filter anything. 28 | /// For example if you want filter preloading segments use: `|e| !(e.contains("preloading"))`. 29 | pub fn new( 30 | request: Request, 31 | http: Client, 32 | filter: Option bool>, 33 | ) -> (Self, UnboundedReceiver) { 34 | let (tx, rx) = unbounded_channel(); 35 | let master_url = request 36 | .url() 37 | .join(".") 38 | .expect("Could not join url with '.'."); 39 | ( 40 | HlsWatch { 41 | tx, 42 | request, 43 | http, 44 | links: PatriciaSet::new(), 45 | master_url, 46 | timeout: Duration::from_secs(10), 47 | fail_counter: 0, 48 | filter, 49 | }, 50 | rx, 51 | ) 52 | } 53 | 54 | pub async fn run(mut self) -> Result<(), Error> { 55 | loop { 56 | if self.fail_counter > HLS_MAX_RETRIES { 57 | // There have either been errors or no new segments 58 | // for `HLS_MAX_RETRIES` times the segment duration given 59 | // in the m3u8 playlist file. 60 | if self.tx.send(HlsQueue::StreamOver).is_err() { 61 | return Err(Error::TIO(std::io::Error::last_os_error())); 62 | }; 63 | break; 64 | } 65 | 66 | // Clone the request so we can reuse it in the loop. 67 | let req = clone_request(&self.request, self.timeout); 68 | let res = match self.http.execute(req).await { 69 | Ok(r) => r, 70 | Err(e) => { 71 | warn!("[HLS] Playlist download failed!\n{}", e); 72 | self.fail_counter += 1; 73 | continue; 74 | } 75 | }; 76 | 77 | let m3u8_string = match res.text().await { 78 | Ok(t) => t, 79 | Err(e) => { 80 | warn!("[HLS] Playlist text failed!\n{}", e); 81 | self.fail_counter += 1; 82 | continue; 83 | } 84 | }; 85 | 86 | let mut m3u8_parser = MediaPlaylist::builder(); 87 | 88 | // Allow excess segment duration because a lot of video sites have 89 | // not very high quality m3u8 playlists, where the video segments, 90 | // may be longer than what the file specifies as max. 91 | m3u8_parser.allowable_excess_duration(Duration::from_secs(10)); 92 | 93 | let m3u8 = match m3u8_parser.parse(&m3u8_string) { 94 | Ok(p) => p, 95 | Err(e) => { 96 | warn!("[HLS] Parsing failed!\n{}", e); 97 | trace!("[HLS]\n{}", &m3u8_string); 98 | self.fail_counter += 1; 99 | continue; 100 | } 101 | }; 102 | 103 | // Get the target duration of a segment 104 | let target_duration = m3u8.target_duration; 105 | 106 | // Makes a iterator with the url parts from the playlist 107 | for e in m3u8.segments.iter().map(|(_, e)| e.uri().trim()) { 108 | trace!("[HLS] Tries to inserts: {}", e); 109 | // Check if we have the segment in our set already 110 | if self.links.insert(e) { 111 | // Reset the counter as we got a new segment. 112 | self.fail_counter = 0; 113 | 114 | // Construct a url from the master and the segment. 115 | let url_formatted = if let Ok(u) = Url::parse(e) { 116 | u 117 | } else { 118 | // Attempt to parse the url as a relative url. 119 | Url::parse(&format!("{}{}", self.master_url.as_str(), &e)).expect( 120 | "The m3u8 does not currently work with stream_lib, \ 121 | please report the issue on the github repo, with an \ 122 | example of the file if possible.", 123 | ) 124 | }; 125 | 126 | // Check that the filter runs. 127 | if self.filter.map_or(true, |f| f(e)) { 128 | debug!("[HLS] Adds {}!", url_formatted); 129 | // Add the segment to the queue. 130 | if self.tx.send(HlsQueue::Url(url_formatted)).is_err() { 131 | return Err(Error::TIO(std::io::Error::last_os_error())); 132 | }; 133 | } 134 | } 135 | } 136 | 137 | if m3u8.has_end_list { 138 | tracing::debug!("List has end, no more segments expected."); 139 | break; 140 | } 141 | 142 | trace!("[HLS] Sleeps for {:#?}", target_duration); 143 | // Sleeps for the target duration. 144 | tokio::time::sleep(target_duration).await; 145 | self.fail_counter += 1; 146 | } 147 | 148 | Ok(()) 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /stream_lib/src/hls/mod.rs: -------------------------------------------------------------------------------- 1 | mod named_watch; 2 | mod watch; 3 | 4 | /// HLS will try and look for new segments 12 times, 5 | pub const HLS_MAX_RETRIES: usize = 12; 6 | 7 | use std::time::Duration; 8 | 9 | use reqwest::header::HeaderMap; 10 | use reqwest::{Client, Method, Request, Url}; 11 | 12 | use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; 13 | 14 | use tracing::warn; 15 | 16 | use futures_util::StreamExt; 17 | 18 | use crate::download_stream::{DownloadStream, Event}; 19 | use crate::error::Error; 20 | 21 | use watch::HlsWatch; 22 | 23 | use named_watch::NamedHlsWatch; 24 | 25 | #[derive(Debug, Clone)] 26 | pub enum HlsQueue { 27 | Url(Url), 28 | StreamOver, 29 | } 30 | pub struct HlsDownloader { 31 | http: Client, 32 | rx: UnboundedReceiver, 33 | watch: Watcher, 34 | headers: HeaderMap, 35 | } 36 | 37 | enum Watcher { 38 | Unnamed(HlsWatch), 39 | Named(NamedHlsWatch), 40 | } 41 | 42 | impl Watcher { 43 | async fn run(self) -> Result<(), Error> { 44 | match self { 45 | Watcher::Unnamed(watch) => watch.run().await, 46 | Watcher::Named(watch) => watch.run().await, 47 | } 48 | } 49 | } 50 | 51 | impl HlsDownloader { 52 | pub(crate) fn new(request: Request, http: Client, filter: Option bool>) -> Self { 53 | let headers = request.headers().clone(); 54 | let (watch, rx) = HlsWatch::new(request, http.clone(), filter); 55 | Self { 56 | http, 57 | rx, 58 | watch: Watcher::Unnamed(watch), 59 | headers, 60 | } 61 | } 62 | 63 | pub(crate) fn new_named( 64 | request: Request, 65 | http: Client, 66 | name: String, 67 | filter: Option bool>, 68 | ) -> Self { 69 | let headers = request.headers().clone(); 70 | let (watch, rx) = NamedHlsWatch::new(request, http.clone(), name, filter); 71 | Self { 72 | http, 73 | rx, 74 | watch: Watcher::Named(watch), 75 | headers, 76 | } 77 | } 78 | 79 | pub(crate) fn new_master_first( 80 | request: Request, 81 | http: Client, 82 | filter: Option bool>, 83 | ) -> Self { 84 | let headers = request.headers().clone(); 85 | let (watch, rx) = NamedHlsWatch::new_first(request, http.clone(), filter); 86 | Self { 87 | http, 88 | rx, 89 | watch: Watcher::Named(watch), 90 | headers, 91 | } 92 | } 93 | 94 | pub(crate) fn download(self) -> DownloadStream { 95 | let rx = self.rx; 96 | let watch = self.watch; 97 | 98 | // TODO: Maybe clean this up after closing the function. 99 | tokio::task::spawn(watch.run()); 100 | 101 | let (download_stream, event_tx) = DownloadStream::new(); 102 | 103 | tokio::task::spawn(bytes_forwarder(self.http, self.headers, rx, event_tx)); 104 | 105 | download_stream 106 | } 107 | } 108 | 109 | async fn bytes_forwarder( 110 | http: Client, 111 | headers: HeaderMap, 112 | mut hls_rx: UnboundedReceiver, 113 | event_tx: UnboundedSender, 114 | ) { 115 | while let Some(hls) = hls_rx.recv().await { 116 | //println!("GOT ELEMENT"); 117 | match hls { 118 | HlsQueue::Url(u) => { 119 | // These two statements are not part of the spinner. 120 | const TIMEOUT: Duration = Duration::from_secs(10); 121 | let req = http 122 | .get(u) 123 | .headers(headers.clone()) 124 | .timeout(TIMEOUT) 125 | .build() 126 | .unwrap(); 127 | if let Err(error) = download_to_file( 128 | http.clone(), 129 | req, 130 | event_tx.clone(), 131 | Some(TIMEOUT), 132 | Some(TIMEOUT), 133 | ) 134 | .await 135 | { 136 | if let Err(error) = event_tx.send(Event::Error { error }) { 137 | warn!("Could not send event: {}", error); 138 | }; 139 | } 140 | } 141 | HlsQueue::StreamOver => { 142 | if let Err(error) = event_tx.send(Event::End) { 143 | warn!("Could not send event: {}", error); 144 | }; 145 | break; 146 | } 147 | } 148 | } 149 | } 150 | 151 | /// Timeout is the total duration the response may take set to none to make it unlimited. 152 | /// Download timeout is the timeout between two chunks of a streaming response. 153 | pub(crate) async fn download_to_file( 154 | client: Client, 155 | mut request: Request, 156 | event_tx: UnboundedSender, 157 | timeout: Option, 158 | download_timeout: Option, 159 | ) -> Result<(), Error> { 160 | const TIMEOUT_MAX: Duration = Duration::MAX; 161 | 162 | *request.timeout_mut() = timeout; 163 | let mut stream = client.execute(request).await?.bytes_stream(); 164 | while let Ok(Some(item)) = 165 | tokio::time::timeout(download_timeout.unwrap_or(TIMEOUT_MAX), stream.next()).await 166 | { 167 | match item { 168 | Ok(bytes) => { 169 | if let Err(error) = event_tx.send(Event::Bytes { bytes }) { 170 | warn!("Could not send event: {}", error); 171 | }; 172 | } 173 | Err(error) => { 174 | if let Err(error) = event_tx.send(Event::Error { 175 | error: error.into(), 176 | }) { 177 | warn!("Could not send event: {}", error); 178 | }; 179 | } 180 | } 181 | } 182 | 183 | Ok(()) 184 | } 185 | 186 | pub fn clone_request(request: &Request, timeout: Duration) -> Request { 187 | if let Some(mut r) = request.try_clone() { 188 | *r.timeout_mut() = Some(timeout); 189 | r 190 | } else { 191 | warn!("[HLS] body not able to be cloned only clones headers."); 192 | let mut r = Request::new(Method::GET, request.url().clone()); 193 | *r.headers_mut() = request.headers().clone(); 194 | *r.timeout_mut() = Some(timeout); 195 | r 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /rsget_cli/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | path::{Path, PathBuf}, 3 | process::Command, 4 | time::Duration, 5 | }; 6 | 7 | use clap::Parser; 8 | use futures_util::StreamExt as _; 9 | use reqwest::Url; 10 | use rsget_lib::{ 11 | utils::error::{RsgetError, StreamError, StreamResult}, 12 | Status, Streamable, 13 | }; 14 | use stream_lib::Event; 15 | use tokio::{ 16 | fs::File, 17 | io::{AsyncWriteExt as _, BufWriter}, 18 | runtime::Runtime, 19 | }; 20 | use tracing::warn; 21 | 22 | #[derive(Parser, Debug)] 23 | #[command(author, version, about, long_about = None)] 24 | struct Opt { 25 | #[arg(short = 'p', long = "play")] 26 | play: bool, 27 | #[arg(short = 'i', long = "info")] 28 | info: bool, 29 | #[arg(short = 'O', long = "folder", default_value = "./")] 30 | folder: PathBuf, 31 | #[arg(short = 'o', long = "output")] 32 | filename: Option, 33 | #[arg(short = 'n', long = "network-play")] 34 | network_play: bool, 35 | url: String, 36 | } 37 | 38 | fn main() -> StreamResult<()> { 39 | let runtime = Runtime::new()?; 40 | runtime.block_on(async move { async_main().await })?; 41 | runtime.shutdown_timeout(Duration::from_millis(100)); 42 | Ok(()) 43 | } 44 | 45 | async fn async_main() -> StreamResult<()> { 46 | tracing_subscriber::fmt::init(); 47 | 48 | let opt = Opt::parse(); 49 | let url = opt.url; 50 | let parsed_url = Url::parse(&url).unwrap(); 51 | let stream: Box = rsget_lib::utils::sites::get_site(&url).await?; 52 | 53 | match stream.is_online().await? { 54 | Status::Offline => return Err(StreamError::Rsget(RsgetError::Offline)), 55 | Status::Online => (), 56 | Status::Unknown => { 57 | warn!("Not sure if stream is online, but will try"); 58 | } 59 | } 60 | 61 | if opt.info { 62 | println!("{:#?}", stream.get_stream().await?); 63 | return Ok(()); 64 | } 65 | 66 | if opt.play && !opt.network_play { 67 | let status = Command::new("mpv") 68 | .arg("--no-ytdl") 69 | .arg(parsed_url.as_str()) 70 | .status() 71 | .expect("Mpv failed to start"); 72 | std::process::exit(status.code().unwrap()) 73 | } 74 | 75 | /* 76 | if opt.network_play { 77 | let child = tokio::spawn(stream_network(stream)); 78 | if opt.play { 79 | Command::new("mpv") 80 | .arg("--no-ytdl") 81 | .arg("--cache=8192") 82 | .arg("tcp://127.0.0.1:61337") 83 | .status() 84 | .expect("Mpv failed to start"); 85 | } else { 86 | println!("Connect player to "); 87 | } 88 | let _ = child.await; 89 | Ok(()) 90 | } else { 91 | */ 92 | 93 | let folder = opt.folder; 94 | if !folder.is_dir() { 95 | eprintln!( 96 | "The --folder argument was not a directonary. ({:?})", 97 | folder 98 | ); 99 | return Ok(()); 100 | } 101 | 102 | if opt.filename.as_ref().is_some_and(|f| f == "-") { 103 | stream_to_stdout(stream).await?; 104 | return Ok(()); 105 | } 106 | 107 | let file_name = strip_characters( 108 | &opt.filename.unwrap_or(stream.get_default_name().await?), 109 | "<>:\"/\\|?*\0", 110 | ); 111 | let full_path = folder.join(file_name); 112 | let path = Path::new(&full_path); 113 | let mut file = BufWriter::new(File::create(path).await?); 114 | 115 | let mut dl = stream.get_stream().await?; 116 | 117 | let spinsty = indicatif::ProgressStyle::default_spinner() 118 | .template( 119 | "{spinner} Elapsed time: {elapsed_precise}, {.blue}Total download: {bytes:30.yellow}", 120 | ) 121 | .unwrap(); 122 | let spinner = indicatif::ProgressBar::new_spinner().with_style(spinsty); 123 | 124 | let mut size = 0; 125 | 126 | while let Some(event) = dl.next().await { 127 | match event { 128 | Event::Bytes { mut bytes } => { 129 | spinner.inc(bytes.len() as u64); 130 | size += bytes.len(); 131 | file.write_all_buf(&mut bytes).await?; 132 | } 133 | Event::End => { 134 | eprintln!("End received"); 135 | break; 136 | } 137 | Event::Error { error } => { 138 | eprintln!("Error occured when downloading stream: {}", error); 139 | break; 140 | } 141 | } 142 | } 143 | 144 | println!("Downloaded: {} MB", size as f64 / 1000.0 / 1000.0); 145 | Ok(()) 146 | } 147 | 148 | async fn stream_to_stdout(stream: Box) -> Result<(), StreamError> 149 | where 150 | S: Streamable + Send + ?Sized, 151 | { 152 | let mut file = BufWriter::new(tokio::io::stdout()); 153 | let mut dl = stream.get_stream().await?; 154 | 155 | while let Some(event) = dl.next().await { 156 | match event { 157 | Event::Bytes { mut bytes } => { 158 | file.write_all_buf(&mut bytes).await?; 159 | } 160 | Event::End => { 161 | eprintln!("End received"); 162 | break; 163 | } 164 | Event::Error { error } => { 165 | eprintln!("Error occured when downloading stream: {}", error); 166 | break; 167 | } 168 | } 169 | } 170 | 171 | Ok(()) 172 | } 173 | 174 | /* 175 | #[allow(clippy::boxed_local)] 176 | async fn stream_network(stream: Box) -> Result 177 | where 178 | S: Streamable + Send + ?Sized, 179 | { 180 | use tokio::net::TcpListener; 181 | let mut listener = TcpListener::bind("127.0.0.1:61337").await?; 182 | let socket = match listener.accept().await { 183 | Ok((socket, _addr)) => Box::new(socket), 184 | Err(e) => return Err(e.into()), 185 | }; 186 | println!("Starts download!"); 187 | let st = stream.get_stream().await?; 188 | let dl = stream_lib::Stream::new(st); 189 | let http = reqwest::Client::new(); 190 | let size = dl.write_file(&http, socket).await?; 191 | Ok(size) 192 | } 193 | */ 194 | 195 | /// Strip characters not allowed on some file systems such as Posix 196 | /// and NTFS. 197 | fn strip_characters(original: &str, to_strip: &str) -> String { 198 | original 199 | .chars() 200 | .filter(|&c| !to_strip.contains(c)) 201 | .collect() 202 | } 203 | -------------------------------------------------------------------------------- /rsget_lib/src/plugins/twitch.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused)] 2 | 3 | use std::env; 4 | 5 | use chrono::prelude::*; 6 | use hls_m3u8::MasterPlaylist; 7 | use rand::{rngs::SmallRng, Rng, SeedableRng}; 8 | use regex::Regex; 9 | 10 | use std::{ 11 | convert::TryFrom, 12 | time::{SystemTime, UNIX_EPOCH}, 13 | }; 14 | 15 | use stream_lib::DownloadStream; 16 | 17 | use crate::utils::error::RsgetError; 18 | use crate::utils::error::StreamError; 19 | use crate::utils::error::StreamResult; 20 | use crate::{Status, Streamable}; 21 | 22 | use async_trait::async_trait; 23 | 24 | const TWITCH_CLIENT_ID: &str = "fmdejdpeuc71dz6i5q24kpz8kiiynv"; 25 | // The reason we need to use this is explained here: 26 | // https://github.com/streamlink/streamlink/issues/2680#issuecomment-557605851 27 | const TWITCH_CLIENT_ID_PRIVATE: &str = "kimne78kx3ncx6brgo4mv6wki5h1ko"; 28 | 29 | #[derive(Serialize, Deserialize, Debug)] 30 | pub struct StreamPayload { 31 | pub data: Vec, 32 | pub pagination: Pagination, 33 | } 34 | 35 | #[derive(Serialize, Deserialize, Debug)] 36 | pub struct StreamData { 37 | pub id: String, 38 | pub user_id: String, 39 | pub user_name: String, 40 | pub game_id: String, 41 | #[serde(rename = "type")] 42 | pub datum_type: String, 43 | pub title: String, 44 | pub viewer_count: i64, 45 | pub started_at: String, 46 | pub language: String, 47 | pub thumbnail_url: String, 48 | } 49 | 50 | #[derive(Serialize, Deserialize, Debug)] 51 | pub struct Pagination { 52 | pub cursor: Option, 53 | } 54 | 55 | #[derive(Serialize, Deserialize, Debug, Clone)] 56 | pub struct AccessToken { 57 | token: String, 58 | sig: String, 59 | } 60 | 61 | #[derive(Debug, Clone)] 62 | pub struct Twitch { 63 | client: reqwest::Client, 64 | username: String, 65 | url: String, 66 | client_id: String, 67 | access_token: Option, 68 | } 69 | 70 | #[async_trait] 71 | impl Streamable for Twitch { 72 | async fn new(url: String) -> StreamResult> { 73 | let client = reqwest::Client::new(); 74 | let username_re = Regex::new(r"^(?:https?://)?(?:www\.)?twitch\.tv/([a-zA-Z0-9]+)")?; 75 | let cap = username_re.captures(&url).ok_or_else(|| { 76 | StreamError::Rsget(RsgetError::new("[Twitch] Cannot capture username")) 77 | })?; 78 | 79 | let client_id = match env::var("RSGET_TWITCH_CLIENT_ID") { 80 | Ok(val) => val, 81 | Err(_) => String::from(TWITCH_CLIENT_ID), 82 | }; 83 | 84 | let access_token = match env::var("RSGET_TWITCH_ACCESS_TOKEN") { 85 | Ok(val) => Some(val), 86 | Err(_) => None, 87 | }; 88 | 89 | let twitch = Twitch { 90 | client, 91 | username: String::from(&cap[1]), 92 | url: url.clone(), 93 | client_id, 94 | access_token, 95 | }; 96 | 97 | Ok(Box::new(twitch)) 98 | } 99 | async fn get_title(&self) -> StreamResult { 100 | if let Some(token) = &self.access_token { 101 | let stream_url = format!( 102 | "https://api.twitch.tv/helix/streams?user_login={}", 103 | self.username 104 | ); 105 | let payload: StreamPayload = self 106 | .client 107 | .get(&stream_url) 108 | .header("Client-ID", &self.client_id) 109 | .bearer_auth(token) 110 | .send() 111 | .await? 112 | .json() 113 | .await 114 | .map_err(|e| { 115 | println!("{}", e); 116 | e 117 | })?; 118 | 119 | match payload.data.first() { 120 | Some(data) => Ok(data.title.clone()), 121 | None => Err(StreamError::Rsget(RsgetError::new( 122 | "[Twitch] User is offline", 123 | ))), 124 | } 125 | } else { 126 | println!("Access token is not set please complete this flow and set the environment variable RSGET_TWITCH_ACCESS_TOKEN with the value of the access_token after the redirect and rerun"); 127 | let oauth_url = format!( 128 | "https://id.twitch.tv/oauth2/authorize?client_id={}&redirect_uri={}&response_type=token+id_token&scope=openid", 129 | self.client_id, 130 | "http://localhost", 131 | ); 132 | 133 | webbrowser::open(&oauth_url).unwrap(); 134 | 135 | Err(StreamError::Rsget(RsgetError::new( 136 | "[Twitch] No access token is set", 137 | ))) 138 | } 139 | } 140 | async fn get_author(&self) -> StreamResult { 141 | Ok(self.username.clone()) 142 | } 143 | async fn is_online(&self) -> StreamResult { 144 | if self.get_title().await.is_ok() { 145 | Ok(Status::Online) 146 | } else { 147 | Ok(Status::Offline) 148 | } 149 | } 150 | async fn get_stream(&self) -> StreamResult { 151 | let auth_endpoint = format!( 152 | "https://api.twitch.tv/api/channels/{}/access_token?client_id={}", 153 | self.username, TWITCH_CLIENT_ID_PRIVATE 154 | ); 155 | let auth_res = self 156 | .client 157 | .get(auth_endpoint.as_str()) 158 | .send() 159 | .await? 160 | .text() 161 | .await?; 162 | let acs: AccessToken = serde_json::from_str(auth_res.as_str())?; 163 | 164 | let time = SystemTime::now() 165 | .duration_since(UNIX_EPOCH) 166 | .unwrap() 167 | .as_secs(); 168 | let mut rng = SmallRng::seed_from_u64(time); 169 | let playlist_url = format!("https://usher.ttvnw.net/api/channel/hls/{}.m3u8?player=twitchweb&token={}&sig={}&allow_audio_only=true&allow_source=true&type=any&p={}", 170 | self.username, acs.token, acs.sig, rng.gen_range(1..=999_999)); 171 | 172 | let playlist_res = self 173 | .client 174 | .get(playlist_url.as_str()) 175 | .send() 176 | .await? 177 | .text() 178 | .await?; 179 | let playlist = MasterPlaylist::try_from(playlist_res.as_str())?; 180 | let qu_name = playlist.media.first().unwrap().name(); 181 | 182 | Ok(stream_lib::download_hls_named( 183 | self.client.clone(), 184 | self.client.get(&playlist_url).build()?, 185 | String::from(qu_name.as_ref()), 186 | None, 187 | )) 188 | } 189 | async fn get_ext(&self) -> StreamResult { 190 | Ok(String::from("mp4")) 191 | } 192 | 193 | async fn get_default_name(&self) -> StreamResult { 194 | let local: DateTime = Local::now(); 195 | Ok(format!( 196 | "{}-{:04}-{:02}-{:02}-{:02}-{:02}.{}", 197 | self.get_author().await?, 198 | local.year(), 199 | local.month(), 200 | local.day(), 201 | local.hour(), 202 | local.minute(), 203 | self.get_ext().await? 204 | )) 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /rsget_lib/src/plugins/vlive.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused)] 2 | 3 | use crate::{Status, Streamable}; 4 | use regex::Regex; 5 | 6 | use async_trait::async_trait; 7 | 8 | use stream_lib::DownloadStream; 9 | 10 | use crate::utils::error::StreamResult; 11 | 12 | use crate::utils::error::RsgetError; 13 | use crate::utils::error::StreamError; 14 | 15 | #[derive(Debug, Clone)] 16 | pub struct Vlive { 17 | http: reqwest::Client, 18 | url: String, 19 | title: String, 20 | author: String, 21 | video_url: Option, 22 | // TODO FOR ERK: This field is currently unused. This is due to Rsgets design being too focused on making plugin 23 | // implementation easier for developers, but at the expense of more "native" per site support. To access the m3u8 24 | // files and the .ts files from vlive you need to provide a session key for the requests. If you look at where I73 25 | 26 | // define VideoInfo, theres is a list field `streams`. Each of these streams has field "key" which has a name and 27 | // a value, which must be appended as a url parameter to every request to that stream. For example: 28 | // { 29 | // "type": "HLS", 30 | // "key": { 31 | // "type": "param", 32 | // "name": "__gda__", 33 | // "value": "1568079266_45ce9b5ed235565dae29acec5cf9d26f" 34 | // }, 35 | // "source": "https://globalv-rmcnmv.akamaized.net/c/read/v2/VOD_ALPHA/global_v_2019_09_08_228/hls/97753681-d218-11e9-bed2-246e963a41ed.m3u8" 36 | // } 37 | // 38 | // means we must append `?__gda__=156807...26f` to the end of that m3u8. You might think that's not too bad? I actually 39 | // already do that in my code, but as it turns out, the code must **also** be appended to every url in that m3u8 file. 40 | // this is where the weakness of Rsgets design comes in. In `get_stream` you can only return a HLS, Chunked or 41 | // NamedPlaylist, which none (as far as I can see) can be used after you have manually parsed and edited the M3U file. 42 | // Even if you managed to edit this m3u8 file, all the listed .ts files must have the key too! All in all, it means 43 | // that the vlive backend cannot use the recommended stream endpoint but rather relies on manually choosing one of 44 | // the `videos` entries, which I think might not be available for every VOD/Stream 45 | #[allow(unused)] 46 | stream_url: Option, 47 | } 48 | 49 | #[async_trait] 50 | impl Streamable for Vlive { 51 | async fn new(url: String) -> StreamResult> { 52 | let http = reqwest::Client::new(); 53 | 54 | let page_req = http.get(&url).send().await?; 55 | let page = page_req.text().await?; 56 | 57 | // The session key and video ID (not the short video seq id at the end of url) are parsed from js 58 | let vid_id_re = Regex::new(r#"vlive\.video\.init\((\s*"(.*?)",\s*?){6}"#)?; 59 | let vid_key_re = Regex::new(r#"vlive\.video\.init\((\s*"(.*?)",\s*?){7}"#)?; 60 | let vid_chan_re = Regex::new(r#"gaCname\s*:\s*"(.*?)""#)?; 61 | 62 | let id = vid_id_re 63 | .captures(&page) 64 | .ok_or_else(|| StreamError::Rsget(RsgetError::new("No capture found")))?[2] 65 | .to_string(); 66 | let key = vid_key_re 67 | .captures(&page) 68 | .ok_or_else(|| StreamError::Rsget(RsgetError::new("No capture found")))?[2] 69 | .to_string(); 70 | let chan = vid_chan_re 71 | .captures(&page) 72 | .ok_or_else(|| StreamError::Rsget(RsgetError::new("No capture found")))?[1] 73 | .to_string(); 74 | 75 | let page_req = http.get(&format!("https://global.apis.naver.com/rmcnmv/rmcnmv/vod_play_videoInfo.json?key={}&videoId={}", key, id)).send().await?; 76 | 77 | // all these structs are quite excessive for what we actually need but i want to be ready for the "Quality Update" 78 | // Currently this backend just chooses the video with the highest file size, aka most likely to be highest quality 79 | #[derive(Debug, Deserialize)] 80 | struct VideoInfo { 81 | meta: Meta, 82 | videos: Videos, 83 | streams: Vec, 84 | } 85 | #[derive(Debug, Deserialize)] 86 | struct Meta { 87 | subject: String, 88 | } 89 | #[derive(Debug, Deserialize)] 90 | struct Videos { 91 | list: Vec