├── .gitignore ├── example_scripts └── stream-went-live.sh ├── Cargo.toml ├── scripts └── release-binary.sh ├── src ├── models.rs └── main.rs ├── README.md └── Cargo.lock /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | -------------------------------------------------------------------------------- /example_scripts/stream-went-live.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec notify-send TWITCH "$TWITCH_CHANNEL_NAME went live at $TWITCH_STREAM_CREATED_AT! 3 | Downloading $TWITCH_STREAM_ID" 4 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "twitch-scraper" 3 | version = "0.1.3" 4 | edition = "2018" 5 | 6 | authors = ["Ashkan Kiani "] 7 | readme = "README.md" 8 | homepage = "https://github.com/norcalli/twitch-scraper" 9 | repository = "https://github.com/norcalli/twitch-scraper" 10 | license = "MIT/Apache-2.0" 11 | keywords = ["twitch", "devops"] 12 | description = "A program that helps download live twitch streams." 13 | include = ["src/*", "Cargo.toml", "README.md"] 14 | 15 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 16 | 17 | [dependencies] 18 | reqwest = "0.9.20" 19 | structopt = "0.3.1" 20 | derive_more = "0.15.0" 21 | log = "0.4.8" 22 | env_logger = "0.6.2" 23 | rand = "0.7.0" 24 | serde = {version = "1.0.100", features = ["derive"]} 25 | serde_json = "1.0.40" 26 | -------------------------------------------------------------------------------- /scripts/release-binary.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | SAY_PREFIX="$(basename $0): " 3 | lightred() { echo -e "\033[1;31m$*\033[0m"; } 4 | blue() { echo -e "\033[1;34m$*\033[0m"; } 5 | dump() { echo "$SAY_PREFIX$*" >&2; } 6 | say() { echo "$SAY_PREFIX$(blue "$*")" >&2; } 7 | yell() { echo "$SAY_PREFIX$(lightred "$*")" >&2; } 8 | die() { 9 | yell "$*" 10 | exit 111 11 | } 12 | try() { "$@" || die "cannot $*"; } 13 | asuser() { sudo su - "$1" -c "${*:2}"; } 14 | need_var() { test -n "${!1}" || die "$1 must be defined"; } 15 | need_vars() { for var in "$@"; do need_var $var; done; } 16 | has_bin() { which "$1" 2>&1 >/dev/null; } 17 | need_exe() { has_bin "$1" || die "'$1' not found in PATH"; } 18 | need_bin() { need_exe "$1"; } 19 | strictmode() { set -eo pipefail; } 20 | nostrictmode() { set +eo pipefail; } 21 | say_var() { say "$1 = ${!1}"; } 22 | say_vars() { for var in "$@"; do say_var $var; done; } 23 | yell_var() { yell "$1 = ${!1}"; } 24 | yell_vars() { for var in "$@"; do yell_var $var; done; } 25 | 26 | export SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" 27 | 28 | VERSION=$(cat Cargo.toml | reser -f json | jp -u package.version) 29 | 30 | need_var VERSION 31 | say_var VERSION 32 | 33 | yell "Aight?" 34 | read RESPONSE 35 | need_var RESPONSE 36 | test "$(echo "$RESPONSE" | tr A-Z a-z)" = tight || break 37 | 38 | linux_musl() { 39 | LINUX_MUSL=twitch-scraper-x86_64-linux-musl 40 | docker run --rm -it -v $PWD:/home/rust/src ekidd/rust-musl-builder:stable cargo build --release && 41 | cp target/x86_64-unknown-linux-musl/release/twitch-scraper $LINUX_MUSL && 42 | strip $LINUX_MUSL && 43 | hub release create -e -m "Version v$VERSION" -m "$(sha256sum $LINUX_MUSL)" -a $LINUX_MUSL v$VERSION 44 | 45 | } 46 | 47 | linux_musl 48 | -------------------------------------------------------------------------------- /src/models.rs: -------------------------------------------------------------------------------- 1 | pub mod kraken { 2 | pub mod search { 3 | pub mod channels { 4 | #[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] 5 | pub struct Root { 6 | #[serde(rename = "_total")] 7 | pub total: i64, 8 | pub channels: Vec, 9 | } 10 | 11 | #[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] 12 | pub struct Channel { 13 | #[serde(rename = "_id")] 14 | pub id: i64, 15 | // pub display_name: String, 16 | pub name: String, 17 | pub status: Option, 18 | #[serde(flatten)] 19 | pub rest: serde_json::Value, 20 | } 21 | 22 | } 23 | 24 | } 25 | 26 | pub mod streams { 27 | /* 28 | { 29 | "_total": 1295, 30 | "streams": [ 31 | { 32 | "_id": 23937446096, 33 | "average_fps": 60, 34 | "channel": { 35 | "_id": 121059319, 36 | "broadcaster_language": "en", 37 | "created_at": "2016-04-06T04:12:40Z", 38 | "display_name": "MOONMOON_OW", 39 | "followers": 251220, 40 | "game": "Overwatch", 41 | "language": "en", 42 | "logo": "https://static-cdn.jtvnw.net/jtv_user_pictures/moonmoon_ow-profile_image-0fe586039bb28259-300x300.png", 43 | "mature": true, 44 | "name": "moonmoon_ow", 45 | "partner": true, 46 | "profile_banner": "https://static-cdn.jtvnw.net/jtv_user_pictures/moonmoon_ow-profile_banner-13fbfa1ba07bcd8a-480.png", 47 | "profile_banner_background_color": null, 48 | "status": "KKona where my Darryl subs at KKona", 49 | "updated_at": "2016-12-15T20:04:53Z", 50 | "url": "https://www.twitch.tv/moonmoon_ow", 51 | "video_banner": "https://static-cdn.jtvnw.net/jtv_user_pictures/moonmoon_ow-channel_offline_image-2b3302e20384eee8-1920x1080.png", 52 | "views": 9869754 53 | }, 54 | "created_at": "2016-12-15T14:55:49Z", 55 | "delay": 0, 56 | "game": "Overwatch", 57 | "is_playlist": false, 58 | "preview": { 59 | "large": "https://static-cdn.jtvnw.net/previews-ttv/live_user_moonmoon_ow-640x360.jpg", 60 | "medium": "https://static-cdn.jtvnw.net/previews-ttv/live_user_moonmoon_ow-320x180.jpg", 61 | "small": "https://static-cdn.jtvnw.net/previews-ttv/live_user_moonmoon_ow-80x45.jpg", 62 | "template": "https://static-cdn.jtvnw.net/previews-ttv/live_user_moonmoon_ow-{width}x{height}.jpg" 63 | }, 64 | "video_height": 720, 65 | "viewers": 11523 66 | } 67 | ] 68 | } 69 | */ 70 | pub mod query { 71 | #[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] 72 | pub struct Root { 73 | #[serde(rename = "_total")] 74 | pub total: i64, 75 | pub streams: Vec, 76 | } 77 | 78 | #[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] 79 | pub struct Stream { 80 | #[serde(rename = "_id")] 81 | pub id: i64, 82 | pub channel: crate::models::kraken::search::channels::Channel, 83 | pub created_at: String, 84 | #[serde(flatten)] 85 | pub rest: serde_json::Value, 86 | } 87 | } 88 | } 89 | } 90 | 91 | pub mod helix { 92 | pub mod search { 93 | pub mod channels { 94 | #[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] 95 | pub struct Root { 96 | #[serde(rename = "_total")] 97 | pub total: i64, 98 | pub channels: Vec, 99 | } 100 | 101 | #[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] 102 | pub struct Channel { 103 | #[serde(rename = "_id")] 104 | pub id: i64, 105 | // pub display_name: String, 106 | pub name: String, 107 | pub status: Option, 108 | #[serde(flatten)] 109 | pub rest: serde_json::Value, 110 | } 111 | 112 | } 113 | 114 | } 115 | 116 | /* 117 | { 118 | "data": [ 119 | { 120 | "id": "26007351216", 121 | "user_id": "7236692", 122 | "user_name": "BillyBob", 123 | "game_id": "29307", 124 | "type": "live", 125 | "title": "[Punday Monday] Necromancer - Dan's First Character - Maps - !build", 126 | "viewer_count": 5723, 127 | "started_at": "2017-08-14T15:45:17Z", 128 | "language": "en", 129 | "thumbnail_url": "https://static-cdn.jtvnw.net/previews-ttv/live_user_dansgaming-{width}x{height}.jpg" 130 | } 131 | ], 132 | "pagination": { 133 | "cursor": "eyJiIjp7Ik9mZnNldCI6MH0sImEiOnsiT2Zmc2V0Ijo0MH19" 134 | } 135 | } 136 | json_typegen --options '{ field_visibility: "pub", derives: "Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize" }' 137 | */ 138 | pub mod streams { 139 | #[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] 140 | pub struct Root { 141 | pub data: Vec, 142 | pub pagination: Pagination, 143 | } 144 | 145 | #[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] 146 | pub struct Datum { 147 | pub id: String, 148 | pub user_id: String, 149 | pub user_name: String, 150 | pub game_id: String, 151 | #[serde(rename = "type")] 152 | pub type_field: String, 153 | pub title: String, 154 | pub viewer_count: i64, 155 | pub started_at: String, 156 | pub language: String, 157 | pub thumbnail_url: String, 158 | } 159 | 160 | #[derive(Default, Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] 161 | pub struct Pagination { 162 | pub cursor: Option, 163 | } 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ``` 2 | $$$$$$$$\ $$\ $$\ $$$$$$\ $$$$$$$$\ $$$$$$\ $$\ $$\ 3 | \__$$ __|$$ | $\ $$ |\_$$ _|\__$$ __|$$ __$$\ $$ | $$ | 4 | $$ | $$ |$$$\ $$ | $$ | $$ | $$ / \__|$$ | $$ | 5 | $$ | $$ $$ $$\$$ | $$ | $$ | $$ | $$$$$$$$ | 6 | $$ | $$$$ _$$$$ | $$ | $$ | $$ | $$ __$$ | 7 | $$ | $$$ / \$$$ | $$ | $$ | $$ | $$\ $$ | $$ | 8 | $$ | $$ / \$$ |$$$$$$\ $$ | \$$$$$$ |$$ | $$ | 9 | \__| \__/ \__|\______| \__| \______/ \__| \__| 10 | $$$$$$\ $$$$$$\ $$$$$$$\ $$$$$$\ $$$$$$$\ $$$$$$$$\ $$$$$$$\ 11 | $$ __$$\ $$ __$$\ $$ __$$\ $$ __$$\ $$ __$$\ $$ _____|$$ __$$\ 12 | $$ / \__|$$ / \__|$$ | $$ |$$ / $$ |$$ | $$ |$$ | $$ | $$ | 13 | \$$$$$$\ $$ | $$$$$$$ |$$$$$$$$ |$$$$$$$ |$$$$$\ $$$$$$$ | 14 | \____$$\ $$ | $$ __$$< $$ __$$ |$$ ____/ $$ __| $$ __$$< 15 | $$\ $$ |$$ | $$\ $$ | $$ |$$ | $$ |$$ | $$ | $$ | $$ | 16 | \$$$$$$ |\$$$$$$ |$$ | $$ |$$ | $$ |$$ | $$$$$$$$\ $$ | $$ | 17 | \______/ \______/ \__| \__|\__| \__|\__| \________|\__| \__| 18 | ``` 19 | 20 | 21 | ```sh 22 | ❯ cargo install twitch-scraper # Install via cargo 23 | Updating crates.io index 24 | Downloaded twitch-scraper v0.1.3 25 | ... 26 | 27 | ❯ curl -OL https://github.com/norcalli/twitch-scraper/releases/download/v0.1.3/twitch-scraper \ 28 | && chmod +x twitch-scraper # Or download a prebuilt static binary 29 | 30 | ❯ twitch-scraper \ 31 | -q \ 32 | -c $TWITCH_CLIENT_ID \ 33 | -d /videos/twitch/ \ 34 | -o "%(channel)s/%(title)s-%(id)s.%(ext)s" \ 35 | -x $PWD/stream-went-live.sh \ 36 | ashkankiani naysayer88 demolition_d studio_trigger 37 | # Example session 38 | [2019-09-12T16:26:47Z INFO twitch_scraper] Watching ["demolition_d", "naysayer88", "ashkankiani", "studio_trigger"] 39 | [2019-09-12T16:34:45Z INFO twitch_scraper] Downloading stream 35633084336 from ashkankiani 40 | [2019-09-12T16:36:21Z INFO twitch_scraper] Finished downloading stream 35633084336 41 | [2019-09-13T10:01:47Z INFO twitch_scraper] Downloading stream 35628821840 from naysayer88 42 | ERROR: ffmpeg exited with code 255 43 | [2019-09-13T10:02:19Z ERROR twitch_scraper] Downloading of stream 35628821840 from naysayer88 failed with status ExitStatus(ExitStatus(256)) 44 | [2019-09-13T10:02:19Z INFO twitch_scraper] Downloading stream 35628821840 from naysayer88 45 | [2019-09-13T10:02:49Z ERROR twitch_scraper] Downloading of stream 35628821840 from naysayer88 failed with status ExitStatus(ExitStatus(15)) 46 | [2019-09-13T10:02:50Z INFO twitch_scraper] Downloading stream 35628821840 from naysayer88 47 | [2019-09-13T12:00:47Z INFO twitch_scraper] Finished downloading stream 35628821840 from naysayer88 48 | 49 | ❯ cat example_scripts/stream-went-live.sh 50 | #!/bin/sh 51 | exec notify-send TWITCH "$TWITCH_CHANNEL_NAME went live at $TWITCH_STREAM_CREATED_AT! 52 | Downloading $TWITCH_STREAM_ID" 53 | 54 | 55 | ❯ twitch-scraper --help 56 | twitch-scraper 0.1.3 57 | Program to poll twitch via its API and download streams from channels as they come live. 58 | 59 | Fault tolerance: 60 | - It will retry requests to Twitch API with exponential jitter backoff up to 5s between retries. 61 | - It will handle API limit rating. 62 | - It will try to restart the download in case youtube-dl fails prematurely. 63 | 64 | Sending SIGINT will kill all running downloads. 65 | Sending SIGTERM will keep the downloads running. 66 | 67 | Use RUST_LOG to set logging level. 68 | e.g. export RUST_LOG='debug' or export RUST_LOG='twitch_scraper=info' 69 | 70 | USAGE: 71 | twitch-scraper [FLAGS] [OPTIONS] --client-id --directory [--] [channel-names]... 72 | 73 | FLAGS: 74 | -h, --help 75 | Prints help information 76 | 77 | -q, --quiet 78 | Quiet output for youtube-dl. 79 | 80 | Shortcut for --additional_args=-q. 81 | -V, --version 82 | Prints version information 83 | 84 | 85 | OPTIONS: 86 | -a, --additional-args ... 87 | Extra args to pass to youtube-dl. 88 | 89 | Current arguments are: --write-info-json --hls-use-mpegts --no-part --netrc 90 | -c, --client-id 91 | Twitch client id 92 | 93 | --delay-max 94 | Maximum milliseconds to wait before polling again [default: 3000] 95 | 96 | --delay-min 97 | Minimum milliseconds to wait before polling again [default: 100] 98 | 99 | -d, --directory 100 | Directory to save videos. 101 | 102 | -o, --filename-template 103 | Directory to save videos. 104 | 105 | See `man youtube-dl` under OUTPUT TEMPLATE for variables to use. 106 | 107 | Useful variables: 108 | - %(uploader)s: channel name 109 | - %(uploader_id)s: channel name (lowercase) 110 | - %(description)s: channel status/title 111 | - %(timestamp)s 112 | - %(title)s: for a live stream, looks like 'ashkankiani 2019-09-06 14_19' 113 | 114 | Note that in the case of a temporary download failure, when youtube-dl is restarted the filename may change, 115 | meaning a separate file is created. This occurs with %(title)s, which uses the time of downloading as part 116 | of the filename. 117 | 118 | MPEGTS files can be concatenated together without any complex processing (e.g. with `cat`), so these streams 119 | can be trivially recomposed. 120 | 121 | I personally use "%(uploader)s/%(uploader)s-%(id)s-%(description)s.%(ext)s" 122 | 123 | [default: %(title)s-%(id)s.%(ext)s] 124 | -x, --script