├── .editorconfig ├── .eslintrc.json ├── .github └── dependabot.yml ├── .gitignore ├── LICENSE ├── README.md ├── config ├── template_config.json ├── template_default.json └── test.json ├── package-lock.json ├── package.json ├── public └── favicon.ico ├── src ├── app.hooks.js ├── app.js ├── check.js ├── google.js ├── index.js ├── logger.js ├── middleware │ ├── admin.js │ ├── dmca.js │ ├── drive.js │ ├── emotes.js │ ├── ffmpeg.js │ ├── index.js │ ├── kick.js │ ├── live.js │ ├── logs.js │ ├── rateLimit.js │ ├── twitch.js │ ├── vod.js │ └── youtube.js ├── models │ ├── emotes.model.js │ ├── games.model.js │ ├── logs.model.js │ ├── streams.model.js │ └── vods.model.js ├── redis.js ├── sequelize.js └── services │ ├── cache.js │ ├── emotes │ ├── emotes.class.js │ ├── emotes.hooks.js │ └── emotes.service.js │ ├── games │ ├── games.class.js │ ├── games.hooks.js │ ├── games.service.js │ └── include.js │ ├── index.js │ ├── logs │ ├── logs.class.js │ ├── logs.hooks.js │ ├── logs.service.js │ └── modify.js │ ├── streams │ ├── streams.class.js │ ├── streams.hooks.js │ └── streams.service.js │ └── vods │ ├── include.js │ ├── vods.class.js │ ├── vods.hooks.js │ └── vods.service.js └── test ├── app.test.js └── services ├── emotes.test.js ├── games.test.js ├── logs.test.js ├── streams.test.js └── vods.test.js /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "es6": true, 4 | "node": true, 5 | "mocha": true 6 | }, 7 | "parserOptions": { 8 | "ecmaVersion": 2018 9 | }, 10 | "extends": [ 11 | "eslint:recommended" 12 | ], 13 | "rules": { 14 | "indent": [ 15 | "error", 16 | 2 17 | ], 18 | "linebreak-style": [ 19 | "error", 20 | "unix" 21 | ], 22 | "quotes": [ 23 | "error", 24 | "single" 25 | ], 26 | "semi": [ 27 | "error", 28 | "always" 29 | ] 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | open-pull-requests-limit: 10 8 | ignore: 9 | - dependency-name: googleapis 10 | versions: 11 | - 67.0.0 12 | - 67.1.0 13 | - 67.1.1 14 | - 68.0.0 15 | - 70.0.0 16 | - 71.0.0 17 | - 72.0.0 18 | - dependency-name: eslint 19 | versions: 20 | - 7.18.0 21 | - 7.19.0 22 | - 7.20.0 23 | - 7.21.0 24 | - 7.22.0 25 | - 7.23.0 26 | - 7.24.0 27 | - dependency-name: redis 28 | versions: 29 | - 3.1.1 30 | - dependency-name: y18n 31 | versions: 32 | - 4.0.1 33 | - 4.0.2 34 | - 4.0.3 35 | - dependency-name: hls-parser 36 | versions: 37 | - 0.10.0 38 | - 0.10.1 39 | - 0.10.2 40 | - 0.10.3 41 | - 0.8.0 42 | - 0.9.0 43 | - dependency-name: sequelize 44 | versions: 45 | - 6.5.0 46 | - 6.5.1 47 | - 6.6.1 48 | - 6.6.2 49 | - dependency-name: mocha 50 | versions: 51 | - 8.3.0 52 | - 8.3.1 53 | - 8.3.2 54 | - dependency-name: express-rate-limit 55 | versions: 56 | - 5.2.5 57 | - 5.2.6 58 | - dependency-name: feathers-hooks-common 59 | versions: 60 | - 5.0.5 61 | - dependency-name: helmet 62 | versions: 63 | - 4.4.1 64 | - dependency-name: nodemon 65 | versions: 66 | - 2.0.7 67 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | 3 | # Runtime data 4 | pids 5 | *.pid 6 | *.seed 7 | 8 | # Directory for instrumented libs generated by jscoverage/JSCover 9 | lib-cov 10 | 11 | # Coverage directory used by tools like istanbul 12 | coverage 13 | 14 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 15 | .grunt 16 | 17 | # Compiled binary addons (http://nodejs.org/api/addons.html) 18 | build/Release 19 | 20 | # Dependency directory 21 | # Commenting this out is preferred by some people, see 22 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 23 | node_modules 24 | 25 | # Users Environment Variables 26 | .lock-wscript 27 | 28 | # IDEs and editors (shamelessly copied from @angular/cli's .gitignore) 29 | /.idea 30 | .project 31 | .classpath 32 | .c9/ 33 | *.launch 34 | .settings/ 35 | *.sublime-workspace 36 | 37 | # IDE - VSCode 38 | .vscode/* 39 | !.vscode/tasks.json 40 | !.vscode/launch.json 41 | !.vscode/extensions.json 42 | 43 | ### Linux ### 44 | *~ 45 | 46 | # temporary files which can be created if a process still has a handle open of a deleted file 47 | .fuse_hidden* 48 | 49 | # KDE directory preferences 50 | .directory 51 | 52 | # Linux trash folder which might appear on any partition or disk 53 | .Trash-* 54 | 55 | # .nfs files are created when an open file is removed but is still being accessed 56 | .nfs* 57 | 58 | ### OSX ### 59 | *.DS_Store 60 | .AppleDouble 61 | .LSOverride 62 | 63 | # Icon must end with two \r 64 | Icon 65 | 66 | 67 | # Thumbnails 68 | ._* 69 | 70 | # Files that might appear in the root of a volume 71 | .DocumentRevisions-V100 72 | .fseventsd 73 | .Spotlight-V100 74 | .TemporaryItems 75 | .Trashes 76 | .VolumeIcon.icns 77 | .com.apple.timemachine.donotpresent 78 | 79 | # Directories potentially created on remote AFP share 80 | .AppleDB 81 | .AppleDesktop 82 | Network Trash Folder 83 | Temporary Items 84 | .apdisk 85 | 86 | ### Windows ### 87 | # Windows thumbnail cache files 88 | Thumbs.db 89 | ehthumbs.db 90 | ehthumbs_vista.db 91 | 92 | # Folder config file 93 | Desktop.ini 94 | 95 | # Recycle Bin used on file shares 96 | $RECYCLE.BIN/ 97 | 98 | # Windows Installer files 99 | *.cab 100 | *.msi 101 | *.msm 102 | *.msp 103 | 104 | # Windows shortcuts 105 | *.lnk 106 | 107 | # Others 108 | lib/ 109 | data/ 110 | 111 | config/* 112 | !config/template_config.json 113 | !config/template_default.json -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Overpowered 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Archives 2 | 3 | > 4 | 5 | ## About 6 | 7 | Automated Upload Twitch Vod to Youtube after streaming 8 | 9 | ## Getting Started 10 | 11 | Getting up and running is as easy as 1, 2, 3. 12 | 13 | 1. Make sure you have [NodeJS](https://nodejs.org/) and [npm](https://www.npmjs.com/) installed. 14 | 2. Install your dependencies 15 | 16 | ``` 17 | cd path/to/archive 18 | npm install 19 | install postgresql 20 | ``` 21 | 22 | 3. Add tables using the src/services folder. 23 | For example: There is a logs service and a vods service. Add them both as a table in postgres. 24 | 25 | 4. Add the columns using the src/models folder. 26 | 27 | 4. Make the following configs using the templates found in the config folder. 28 | 29 | ``` 30 | path/to/archive/config/config.json 31 | path/to/archive/config/default.json 32 | path/to/archive/config/production.json (used in nodejs production mode) (copy default.json) 33 | ``` 34 | 35 | 5. Start your app 36 | 37 | ``` 38 | npm start 39 | ``` 40 | ## Verifying your Youtube channel 41 | 42 | To upload 15 minutes+ videos, you will need to verify your youtube using this [link](https://www.youtube.com/verify) 43 | 44 | ## Verifying your Google Console API 45 | 46 | To make your videos publicly automatically, you need to undergo an audit from google. 47 | 48 | Please refer to https://developers.google.com/youtube/v3/docs/videos/insert 49 | -------------------------------------------------------------------------------- /config/template_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "twitch": { 3 | "enabled": false, 4 | "auth": { 5 | "client_id": "", 6 | "client_secret": "", 7 | "access_token": "" 8 | }, 9 | "id": "", 10 | "username": "" 11 | }, 12 | "kick": { 13 | "enabled": false, 14 | "username": "", 15 | "id": "" 16 | }, 17 | "google": { 18 | "client_id": "", 19 | "client_secret": "", 20 | "redirect_url": "" 21 | }, 22 | "youtube": { 23 | "description": "", 24 | "public": true, 25 | "vodUpload": true, 26 | "perGameUpload": false, 27 | "restrictedGames": [], 28 | "splitDuration": 10800, 29 | "api_key": "", 30 | "liveUpload": false, 31 | "multiTrack": false, 32 | "upload": true, 33 | "auth": { 34 | "access_token": "", 35 | "scope": "https://www.googleapis.com/auth/youtube.force-ssl https://www.googleapis.com/auth/youtube https://www.googleapis.com/auth/youtube.upload", 36 | "token_type": "Bearer", 37 | "expires_in": 3599, 38 | "refresh_token": "" 39 | } 40 | }, 41 | "channel": "", 42 | "domain_name": "", 43 | "vodPath": "/mnt/vods", 44 | "livePath": "/mnt/live", 45 | "timezone": "America/Chicago", 46 | "chatDownload": false, 47 | "vodDownload": true, 48 | "saveHLS": false, 49 | "saveMP4": false, 50 | "drive": { 51 | "upload": false, 52 | "parents": [], 53 | "auth": { 54 | "access_token": "", 55 | "scope": "https://www.googleapis.com/auth/drive", 56 | "token_type": "Bearer", 57 | "expires_in": 3599, 58 | "refresh_token": "" 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /config/template_default.json: -------------------------------------------------------------------------------- 1 | { 2 | "host": "localhost", 3 | "port": 3030, 4 | "public": "./public/", 5 | "paginate": { 6 | "default": 10, 7 | "max": 50 8 | }, 9 | "postgres": "YOUR_POSTGRES_CONNECTION_URI", 10 | "ADMIN_API_KEY": "YOUR_ADMIN_API_KEY", 11 | "redis": { 12 | "host": "127.0.0.1", 13 | "port": 6379, 14 | "useSocket": false, 15 | "path": "/tmp/redis.sock" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /config/test.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "archive", 3 | "description": "", 4 | "version": "0.0.0", 5 | "homepage": "", 6 | "private": true, 7 | "main": "src", 8 | "keywords": [ 9 | "feathers" 10 | ], 11 | "author": { 12 | "name": "TimIsOverpowered" 13 | }, 14 | "contributors": [], 15 | "bugs": {}, 16 | "directories": { 17 | "lib": "src", 18 | "test": "test/", 19 | "config": "config/" 20 | }, 21 | "engines": { 22 | "node": "^16.0.0", 23 | "npm": ">= 3.0.0" 24 | }, 25 | "scripts": { 26 | "test": "npm run lint && npm run mocha", 27 | "lint": "eslint src/. test/. --config .eslintrc.json --fix", 28 | "dev": "nodemon src/", 29 | "start": "node src/", 30 | "mocha": "mocha test/ --recursive --exit", 31 | "production": "NODE_ENV=production node src", 32 | "production-windows": "set NODE_ENV=production&&node src" 33 | }, 34 | "standard": { 35 | "env": [ 36 | "mocha" 37 | ], 38 | "ignore": [] 39 | }, 40 | "dependencies": { 41 | "@feathersjs/configuration": "^5.0.30", 42 | "@feathersjs/errors": "^5.0.30", 43 | "@feathersjs/express": "^5.0.30", 44 | "@feathersjs/feathers": "^5.0.30", 45 | "@feathersjs/transport-commons": "^5.0.30", 46 | "compression": "^1.7.4", 47 | "cors": "^2.8.5", 48 | "dayjs": "^1.11.13", 49 | "feathers-hooks-common": "^8.2.1", 50 | "feathers-sequelize": "^7.0.3", 51 | "fluent-ffmpeg": "^2.1.3", 52 | "googleapis": "^144.0.0", 53 | "helmet": "^7.1.0", 54 | "hls-parser": "^0.13.3", 55 | "ioredis": "^5.4.1", 56 | "pg": "^8.12.0", 57 | "puppeteer": "^23.3.0", 58 | "puppeteer-extra": "^3.3.6", 59 | "puppeteer-extra-plugin-click-and-wait": "^2.3.3", 60 | "puppeteer-extra-plugin-stealth": "^2.11.2", 61 | "puppeteer-real-browser": "^1.3.7", 62 | "qs": "^6.13.0", 63 | "rate-limiter-flexible": "^5.0.3", 64 | "redis": "^4.7.0", 65 | "selenium-webdriver": "^4.24.0", 66 | "sequelize": "^6.37.3", 67 | "serve-favicon": "^2.5.0", 68 | "winston": "^3.14.2" 69 | }, 70 | "devDependencies": { 71 | "axios": "^1.7.7", 72 | "eslint": "^9.9.1", 73 | "mocha": "^10.7.3", 74 | "nodemon": "^3.1.4" 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TimIsOverpowered/archive/771adc093d6b70eab8a5a622b47bfcf6ea0af138/public/favicon.ico -------------------------------------------------------------------------------- /src/app.hooks.js: -------------------------------------------------------------------------------- 1 | // Application hooks that run for every service 2 | 3 | module.exports = { 4 | before: { 5 | all: [], 6 | find: [], 7 | get: [], 8 | create: [], 9 | update: [], 10 | patch: [], 11 | remove: [] 12 | }, 13 | 14 | after: { 15 | all: [], 16 | find: [], 17 | get: [], 18 | create: [], 19 | update: [], 20 | patch: [], 21 | remove: [] 22 | }, 23 | 24 | error: { 25 | all: [], 26 | find: [], 27 | get: [], 28 | create: [], 29 | update: [], 30 | patch: [], 31 | remove: [] 32 | } 33 | }; 34 | -------------------------------------------------------------------------------- /src/app.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | const favicon = require("serve-favicon"); 3 | const compress = require("compression"); 4 | const helmet = require("helmet"); 5 | const cors = require("cors"); 6 | const logger = require("./logger"); 7 | 8 | const { feathers } = require("@feathersjs/feathers"); 9 | const configuration = require("@feathersjs/configuration"); 10 | const express = require("@feathersjs/express"); 11 | 12 | const middleware = require("./middleware"); 13 | const services = require("./services"); 14 | const appHooks = require("./app.hooks"); 15 | const redis = require("./redis"); 16 | const google = require("./google"); 17 | 18 | const sequelize = require("./sequelize"); 19 | 20 | const app = express(feathers()); 21 | 22 | // Load app configuration 23 | app.configure(configuration()); 24 | app.configure(redis); 25 | google.initializeYt(app); 26 | google.initializeDrive(app); 27 | // Enable security, CORS, compression, favicon and body parsing 28 | app.use( 29 | helmet({ 30 | contentSecurityPolicy: false, 31 | }) 32 | ); 33 | app.use(cors()); 34 | app.use(compress()); 35 | app.use(express.json({ limit: "25mb" })); 36 | app.use(express.urlencoded({ extended: true })); 37 | const rawBodySaver = function (req, res, buf, encoding) { 38 | if (buf && buf.length) { 39 | req.rawBody = buf.toString(encoding || "utf8"); 40 | } 41 | }; 42 | app.use(express.raw({ verify: rawBodySaver(), type: "*/*" })); 43 | app.use(favicon(path.join(app.get("public"), "favicon.ico"))); 44 | // Host the public folder 45 | app.use("/", express.static(app.get("public"))); 46 | 47 | // Set up Plugins and providers 48 | app.configure(express.rest()); 49 | 50 | app.configure(sequelize); 51 | 52 | // Configure other middleware (see `middleware/index.js`) 53 | app.configure(middleware); 54 | // Set up our services (see `services/index.js`) 55 | app.configure(services); 56 | 57 | // Configure a middleware for 404s and the error handler 58 | app.use(function (req, res, next) { 59 | res.status(404).json({ error: true, msg: "Missing route" }); 60 | }); 61 | app.use(express.errorHandler({ logger })); 62 | 63 | app.hooks(appHooks); 64 | 65 | module.exports = app; 66 | -------------------------------------------------------------------------------- /src/check.js: -------------------------------------------------------------------------------- 1 | const twitch = require("./middleware/twitch"); 2 | const kick = require("./middleware/kick"); 3 | const config = require("../config/config.json"); 4 | const vod = require("./middleware/vod"); 5 | const emotes = require("./middleware/emotes"); 6 | const fs = require("fs"); 7 | const dayjs = require("dayjs"); 8 | const utc = require("dayjs/plugin/utc"); 9 | dayjs.extend(utc); 10 | 11 | const fileExists = async (file) => { 12 | return fs.promises 13 | .access(file, fs.constants.F_OK) 14 | .then(() => true) 15 | .catch(() => false); 16 | }; 17 | 18 | module.exports.checkTwitch = async (app) => { 19 | const twitchId = config.twitch.id; 20 | const stream = await twitch.getStream(twitchId); 21 | 22 | if (!stream) 23 | return setTimeout(() => { 24 | this.checkTwitch(app); 25 | }, 30000); 26 | 27 | if (!stream[0]) 28 | return setTimeout(() => { 29 | this.checkTwitch(app); 30 | }, 30000); 31 | 32 | const streamExists = await app 33 | .service("streams") 34 | .get(stream.id) 35 | .then(() => true) 36 | .catch(() => false); 37 | 38 | if (!streamExists) 39 | await app 40 | .service("streams") 41 | .create({ 42 | id: stream[0].id, 43 | started_at: stream[0].started_at, 44 | platform: "twitch", 45 | is_live: true, 46 | }) 47 | .then(() => 48 | console.log( 49 | `${config.channel} twitch stream online. Created Stream. ${stream[0].started_at}` 50 | ) 51 | ) 52 | .catch((e) => { 53 | console.error(e); 54 | }); 55 | 56 | const vodData = await twitch.getLatestVodData(twitchId); 57 | 58 | if (!vodData) 59 | return setTimeout(() => { 60 | this.checkTwitch(app); 61 | }, 30000); 62 | 63 | if (vodData.stream_id !== stream[0].id) 64 | return setTimeout(() => { 65 | this.checkTwitch(app); 66 | }, 30000); 67 | 68 | const vodId = vodData.id; 69 | const vodExists = await app 70 | .service("vods") 71 | .get(vodId) 72 | .then(() => true) 73 | .catch(() => false); 74 | 75 | if (!vodExists) { 76 | await app 77 | .service("vods") 78 | .create({ 79 | id: vodId, 80 | title: vodData.title, 81 | createdAt: vodData.created_at, 82 | stream_id: vodData.stream_id, 83 | platform: "twitch", 84 | }) 85 | .then(() => 86 | console.log( 87 | `${config.channel} has a new twitch vod. Creating vod. ${dayjs 88 | .utc(vodData.createdAt) 89 | .format("MM-DD-YYYY")}` 90 | ) 91 | ) 92 | .catch((e) => { 93 | console.error(e); 94 | }); 95 | } 96 | 97 | const vodDownloading = app.get(`${config.channel}-${vodId}-vod-downloading`); 98 | 99 | if (config.vodDownload && !vodDownloading) { 100 | app.set(`${config.channel}-${vodId}-vod-downloading`, true); 101 | const dir = `${config.vodPath}/${vodId}`; 102 | if (await fileExists(dir)) 103 | await fs.promises.rm(dir, { 104 | recursive: true, 105 | }); 106 | console.info(`Start Vod download: ${vodId}`); 107 | vod.download(vodId, app, 0, 1, true); 108 | } 109 | 110 | const chatDownloading = app.get( 111 | `${config.channel}-${vodId}-chat-downloading` 112 | ); 113 | 114 | if (config.chatDownload && !chatDownloading) { 115 | app.set(`${config.channel}-${vodId}-chat-downloading`, true); 116 | console.info(`Start Logs download: ${vodId}`); 117 | vod.downloadLogs(vodId, app); 118 | emotes.save(vodId, app); 119 | } 120 | 121 | setTimeout(() => { 122 | this.checkTwitch(app); 123 | }, 30000); 124 | }; 125 | 126 | module.exports.checkKick = async (app) => { 127 | const kickChannel = config.kick.username; 128 | const stream = await kick.getStream(app, kickChannel); 129 | 130 | if (!stream) 131 | return setTimeout(() => { 132 | this.checkKick(app); 133 | }, 30000); 134 | 135 | if (!stream.data) 136 | return setTimeout(() => { 137 | this.checkKick(app); 138 | }, 30000); 139 | 140 | const streamData = stream.data; 141 | const streamId = streamData.id.toString(); 142 | 143 | const streamExists = await app 144 | .service("streams") 145 | .get(streamId) 146 | .then(() => true) 147 | .catch(() => false); 148 | 149 | if (!streamExists) 150 | await app 151 | .service("streams") 152 | .create({ 153 | id: streamId, 154 | started_at: streamData.created_at, 155 | platform: "kick", 156 | is_live: true, 157 | }) 158 | .then(() => 159 | console.log( 160 | `${config.channel} kick stream online. Created Stream. ${streamData.created_at}` 161 | ) 162 | ) 163 | .catch((e) => { 164 | console.error(e); 165 | }); 166 | 167 | const vodData = await kick.getVod(app, kickChannel, streamId); 168 | 169 | if (!vodData) 170 | return setTimeout(() => { 171 | this.checkKick(app); 172 | }, 30000); 173 | 174 | const vodExists = await app 175 | .service("vods") 176 | .get(streamId) 177 | .then(() => true) 178 | .catch(() => false); 179 | 180 | if (!vodExists) { 181 | await app 182 | .service("vods") 183 | .create({ 184 | id: streamId, 185 | title: streamData.session_title, 186 | createdAt: streamData.created_at, 187 | platform: "kick", 188 | }) 189 | .then(() => 190 | console.log( 191 | `${config.channel} has a new kick vod. Creating vod. ${dayjs 192 | .utc(streamData.created_at) 193 | .format("MM-DD-YYYY")}` 194 | ) 195 | ) 196 | .catch((e) => { 197 | console.error(e); 198 | }); 199 | } 200 | 201 | const vodDownloading = app.get(`${config.channel}-${streamId}-vod-downloading`); 202 | 203 | if (config.vodDownload && !vodDownloading) { 204 | app.set(`${config.channel}-${streamId}-vod-downloading`, true); 205 | const dir = `${config.vodPath}/${streamId}`; 206 | if (await fileExists(dir)) 207 | await fs.promises.rm(dir, { 208 | recursive: true, 209 | }); 210 | console.info(`Start Vod download: ${streamId}`); 211 | await kick.downloadHLS(streamId, app, vodData.source); 212 | } 213 | 214 | const chatDownloading = app.get( 215 | `${config.channel}-${streamId}-chat-downloading` 216 | ); 217 | 218 | if (config.chatDownload && !chatDownloading) { 219 | app.set(`${config.channel}-${streamId}-chat-downloading`, true); 220 | //console.info(`Start Logs download: ${streamId}`); 221 | //kick.downloadLogs(streamId, app); 222 | emotes.save(streamId, app); 223 | } 224 | 225 | setTimeout(() => { 226 | this.checkKick(app); 227 | }, 30000); 228 | }; -------------------------------------------------------------------------------- /src/google.js: -------------------------------------------------------------------------------- 1 | const config = require("../config/config.json"); 2 | const { google } = require("googleapis"); 3 | const OAuth2 = google.auth.OAuth2; 4 | const fs = require("fs"); 5 | const path = require("path"); 6 | 7 | module.exports.initializeYt = (app) => { 8 | const oauth2Client = new OAuth2(config.google.client_id, config.google.client_secret, config.google.redirect_url); 9 | oauth2Client.on("tokens", (tokens) => { 10 | if (tokens.refresh_token) config.youtube.auth.refresh_token = tokens.refresh_token; 11 | config.youtube.auth.access_token = tokens.access_token; 12 | 13 | fs.writeFile(path.resolve(__dirname, "../config/config.json"), JSON.stringify(config, null, 4), (err) => { 14 | if (err) return console.error(err); 15 | console.info("Refreshed Youtube Token"); 16 | }); 17 | 18 | oauth2Client.setCredentials(config.youtube.auth); 19 | }); 20 | 21 | oauth2Client.setCredentials(config.youtube.auth); 22 | 23 | app.set("ytOauth2Client", oauth2Client); 24 | }; 25 | 26 | module.exports.initializeDrive = (app) => { 27 | const oauth2Client = new OAuth2(config.google.client_id, config.google.client_secret, config.google.redirect_url); 28 | oauth2Client.on("tokens", (tokens) => { 29 | if (tokens.refresh_token) config.drive.auth.refresh_token = tokens.refresh_token; 30 | config.drive.auth.access_token = tokens.access_token; 31 | 32 | fs.writeFile(path.resolve(__dirname, "../config/config.json"), JSON.stringify(config, null, 4), (err) => { 33 | if (err) return console.error(err); 34 | console.info("Refreshed Drive Token"); 35 | }); 36 | 37 | oauth2Client.setCredentials(config.drive.auth); 38 | }); 39 | 40 | oauth2Client.setCredentials(config.drive.auth); 41 | 42 | app.set("driveOauth2Client", oauth2Client); 43 | }; 44 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | const logger = require("./logger"); 3 | const app = require("./app"); 4 | const port = app.get("port"); 5 | const host = app.get("host"); 6 | const { checkTwitch, checkKick } = require("./check"); 7 | const config = require("../config/config.json"); 8 | 9 | process.on("unhandledRejection", (reason, p) => { 10 | logger.error("Unhandled Rejection at: Promise ", p, reason); 11 | console.error("Unhandled Rejection at: Promise ", p, " reason: ", reason); 12 | }); 13 | 14 | app.listen(port).then(async () => { 15 | logger.info(`Feathers app listening on http://${host}:${port}`); 16 | if (config.twitch.enabled) checkTwitch(app); 17 | if (config.kick.enabled) { 18 | let { connect } = await import("puppeteer-real-browser"); 19 | const { page, browser } = await connect({ 20 | headless: "auto", 21 | args: ["--no-sandbox", "--disable-setuid-sandbox"], 22 | connectOption: { 23 | defaultViewport: null, 24 | }, 25 | turnstile: true, 26 | plugins: [require("puppeteer-extra-plugin-click-and-wait")()], 27 | }); 28 | page.setDefaultNavigationTimeout(5 * 60 * 1000); 29 | app.set("puppeteer", browser); 30 | checkKick(app); 31 | } 32 | }); 33 | -------------------------------------------------------------------------------- /src/logger.js: -------------------------------------------------------------------------------- 1 | const { createLogger, format, transports } = require('winston'); 2 | 3 | // Configure the Winston logger. For the complete documentation see https://github.com/winstonjs/winston 4 | const logger = createLogger({ 5 | // To see more detailed errors, change this to 'debug' 6 | level: 'info', 7 | format: format.combine( 8 | format.splat(), 9 | format.simple() 10 | ), 11 | transports: [ 12 | new transports.Console() 13 | ], 14 | }); 15 | 16 | module.exports = logger; 17 | -------------------------------------------------------------------------------- /src/middleware/admin.js: -------------------------------------------------------------------------------- 1 | const vod = require("./vod"); 2 | const twitch = require("./twitch"); 3 | const kick = require("./kick"); 4 | const fs = require("fs"); 5 | const config = require("../../config/config.json"); 6 | const drive = require("./drive"); 7 | const emotes = require("./emotes"); 8 | const dayjs = require("dayjs"); 9 | const duration = require("dayjs/plugin/duration"); 10 | const ffmpeg = require("./ffmpeg"); 11 | dayjs.extend(duration); 12 | 13 | module.exports.verify = function (app) { 14 | return async function (req, res, next) { 15 | if (!req.headers["authorization"]) { 16 | res.status(403).json({ error: true, msg: "Missing auth key" }); 17 | return; 18 | } 19 | 20 | const authKey = req.headers.authorization.split(" ")[1]; 21 | const key = app.get("ADMIN_API_KEY"); 22 | 23 | if (key !== authKey) { 24 | res.status(403).json({ error: true, msg: "Not authorized" }); 25 | return; 26 | } 27 | next(); 28 | }; 29 | }; 30 | 31 | module.exports.download = function (app) { 32 | return async function (req, res, next) { 33 | let { vodId, type, platform, path, m3u8 } = req.body; 34 | if (!vodId) return res.status(400).json({ error: true, msg: "No VodId" }); 35 | if (!type) return res.status(400).json({ error: true, msg: "No type" }); 36 | if (!platform) 37 | return res.status(400).json({ error: true, msg: "No platform" }); 38 | 39 | const exists = await app 40 | .service("vods") 41 | .get(vodId) 42 | .then(() => true) 43 | .catch(() => false); 44 | 45 | if (exists) { 46 | res.status(200).json({ error: false, msg: "Starting download.." }); 47 | emotes.save(vodId, app); 48 | if (m3u8) { 49 | await kick.downloadHLS(vodId, app, m3u8); 50 | return; 51 | } 52 | 53 | vod.upload(vodId, app, path, type); 54 | return; 55 | } 56 | 57 | if (platform === "twitch") { 58 | const vodData = await twitch.getVodData(vodId); 59 | if (!vodData) 60 | return res.status(404).json({ error: true, msg: "No Vod Data" }); 61 | 62 | if (vodData.user_id !== config.twitch.id) 63 | return res.status(400).json({ 64 | error: true, 65 | msg: "This vod belongs to another channel..", 66 | }); 67 | 68 | await app 69 | .service("vods") 70 | .create({ 71 | id: vodData.id, 72 | title: vodData.title, 73 | createdAt: vodData.created_at, 74 | duration: dayjs 75 | .duration(`PT${vodData.duration.toUpperCase()}`) 76 | .format("HH:mm:ss"), 77 | stream_id: vodData.stream_id, 78 | platform: "twitch", 79 | }) 80 | .then(() => { 81 | console.info( 82 | `Created twitch vod ${vodData.id} for ${vodData.user_name}` 83 | ); 84 | }) 85 | .catch((e) => { 86 | console.error(e); 87 | }); 88 | 89 | res.status(200).json({ error: false, msg: "Starting download.." }); 90 | emotes.save(vodId, app); 91 | 92 | const vodPath = await vod.upload(vodId, app, path, type, "twitch"); 93 | if (vodPath) fs.unlinkSync(vodPath); 94 | } else if (platform === "kick") { 95 | const vodData = await kick.getVod(app, config.kick.username, vodId); 96 | if (!vodData) 97 | return res.status(404).json({ error: true, msg: "No Vod Data" }); 98 | 99 | if (vodData.channel_id.toString() !== config.kick.id) 100 | return res.status(400).json({ 101 | error: true, 102 | msg: "This vod belongs to another channel..", 103 | }); 104 | 105 | await app 106 | .service("vods") 107 | .create({ 108 | id: vodData.id.toString(), 109 | title: vodData.session_title, 110 | createdAt: vodData.start_time, 111 | duration: dayjs 112 | .duration(vodData.duration, "milliseconds") 113 | .format("HH:mm:ss"), 114 | stream_id: vodData.video.uuid, 115 | platform: "kick", 116 | }) 117 | .then(() => { 118 | console.info( 119 | `Created kick vod ${vodData.id} for ${config.kick.username}` 120 | ); 121 | }) 122 | .catch((e) => { 123 | console.error(e); 124 | }); 125 | res.status(200).json({ error: false, msg: "Starting download.." }); 126 | emotes.save(vodId, app); 127 | 128 | if (m3u8) { 129 | await kick.downloadHLS(vodId, app, m3u8); 130 | return; 131 | } 132 | 133 | const vodPath = await vod.upload(vodId, app, path, type, "kick"); 134 | if (vodPath) fs.unlinkSync(vodPath); 135 | } 136 | }; 137 | }; 138 | 139 | module.exports.hlsDownload = function (app) { 140 | return async function (req, res, next) { 141 | const { vodId } = req.body; 142 | if (!vodId) return res.status(400).json({ error: true, msg: "No VodId" }); 143 | 144 | const exists = await app 145 | .service("vods") 146 | .get(vodId) 147 | .then(() => true) 148 | .catch(() => false); 149 | 150 | if (exists) { 151 | console.info(`Start Vod download: ${vodId}`); 152 | vod.download(vodId, app); 153 | console.info(`Start Logs download: ${vodId}`); 154 | vod.downloadLogs(vodId, app); 155 | res.status(200).json({ error: false, msg: "Starting download.." }); 156 | return; 157 | } 158 | 159 | const vodData = await twitch.getVodData(vodId); 160 | if (!vodData) 161 | return res.status(404).json({ error: true, msg: "No Vod Data" }); 162 | 163 | if (vodData.user_id !== config.twitch.id) 164 | return res.status(400).json({ 165 | error: true, 166 | msg: "This vod belongs to another channel..", 167 | }); 168 | 169 | await app 170 | .service("vods") 171 | .create({ 172 | id: vodData.id, 173 | title: vodData.title, 174 | createdAt: vodData.created_at, 175 | duration: dayjs 176 | .duration(`PT${vodData.duration.toUpperCase()}`) 177 | .format("HH:mm:ss"), 178 | stream_id: vodData.stream_id, 179 | platform: "twitch", 180 | }) 181 | .then(() => { 182 | console.info(`Created vod ${vodData.id} for ${vodData.user_name}`); 183 | }) 184 | .catch((e) => { 185 | console.error(e); 186 | }); 187 | 188 | console.info(`Start Vod download: ${vodId}`); 189 | vod.download(vodId, app); 190 | console.info(`Start Logs download: ${vodId}`); 191 | vod.downloadLogs(vodId, app); 192 | res.status(200).json({ error: false, msg: "Starting download.." }); 193 | }; 194 | }; 195 | 196 | module.exports.logs = function (app) { 197 | return async function (req, res, next) { 198 | const { vodId, platform } = req.body; 199 | if (!vodId) return res.status(400).json({ error: true, msg: "No VodId" }); 200 | if (!platform) 201 | return res.status(400).json({ error: true, msg: "No Platform" }); 202 | 203 | let total; 204 | app 205 | .service("logs") 206 | .find({ 207 | query: { 208 | $limit: 0, 209 | vod_id: vodId, 210 | }, 211 | }) 212 | .then((data) => { 213 | total = data.total; 214 | }) 215 | .catch((e) => { 216 | console.error(e); 217 | }); 218 | 219 | if (total > 1) 220 | return res.status(400).json({ 221 | error: true, 222 | msg: `Logs already exist for ${vodId}`, 223 | }); 224 | 225 | if (platform === "twitch") { 226 | vod.getLogs(vodId, app); 227 | res.status(200).json({ error: false, msg: "Getting logs.." }); 228 | } else if (platform === "kick") { 229 | const vodData = await kick.getVod(app, config.kick.username, vodId); 230 | kick.downloadLogs( 231 | vodId, 232 | app, 233 | dayjs.utc(vodData.start_time).toISOString(), 234 | vodData.duration 235 | ); 236 | res.status(200).json({ error: false, msg: "Getting logs.." }); 237 | } else { 238 | res.status(400).json({ error: false, msg: "Platform not supported.." }); 239 | } 240 | }; 241 | }; 242 | 243 | module.exports.manualLogs = function (app) { 244 | return async function (req, res, next) { 245 | const { vodId, path } = req.body; 246 | if (!vodId) return res.status(400).json({ error: true, msg: "No VodId" }); 247 | if (!path) return res.status(400).json({ error: true, msg: "No Path" }); 248 | 249 | vod.manualLogs(path, vodId, app); 250 | res.status(200).json({ error: false, msg: "Getting logs.." }); 251 | }; 252 | }; 253 | 254 | module.exports.createVod = function (app) { 255 | return async function (req, res, next) { 256 | const { vodId, title, createdAt, duration, drive, platform } = req.body; 257 | if (vodId == null) 258 | return res 259 | .status(400) 260 | .json({ error: true, msg: "Missing parameter: Vod id" }); 261 | if (!title) 262 | return res 263 | .status(400) 264 | .json({ error: true, msg: "Missing parameter: Title" }); 265 | if (!createdAt) 266 | return res 267 | .status(400) 268 | .json({ error: true, msg: "Missing parameter: CreatedAt" }); 269 | if (!duration) 270 | return res 271 | .status(400) 272 | .json({ error: true, msg: "Missing parameter: Duration" }); 273 | if (!platform) 274 | return res 275 | .status(400) 276 | .json({ error: true, msg: "Missing parameter: platform" }); 277 | 278 | const exists = await app 279 | .service("vods") 280 | .get(vodId) 281 | .then(() => true) 282 | .catch(() => false); 283 | 284 | if (exists) 285 | return res 286 | .status(400) 287 | .json({ error: true, msg: `${vodId} already exists!` }); 288 | 289 | await app 290 | .service("vods") 291 | .create({ 292 | id: vodId, 293 | title: title, 294 | createdAt: createdAt, 295 | duration: duration, 296 | drive: drive ? [drive] : [], 297 | platform: platform, 298 | }) 299 | .then(() => { 300 | console.info(`Created vod ${vodId}`); 301 | res.status(200).json({ error: false, msg: `${vodId} Created!` }); 302 | }) 303 | .catch((e) => { 304 | console.error(e); 305 | res 306 | .status(200) 307 | .json({ error: true, msg: `Failed to create ${vodId}!` }); 308 | }); 309 | }; 310 | }; 311 | 312 | module.exports.deleteVod = function (app) { 313 | return async function (req, res, next) { 314 | const { vodId } = req.body; 315 | if (vodId == null) 316 | return res 317 | .status(400) 318 | .json({ error: true, msg: "Missing parameter: Vod id" }); 319 | 320 | res.status(200).json({ error: false, msg: "Starting deletion process.." }); 321 | 322 | await app 323 | .service("vods") 324 | .remove(vodId) 325 | .then(() => { 326 | console.info(`Deleted vod for ${vodId}`); 327 | }) 328 | .catch((e) => { 329 | console.error(e); 330 | }); 331 | 332 | await app 333 | .service("logs") 334 | .remove(null, { 335 | query: { 336 | vod_id: vodId, 337 | }, 338 | }) 339 | .then(() => { 340 | console.info(`Deleted logs for ${vodId}`); 341 | }) 342 | .catch((e) => { 343 | console.error(e); 344 | }); 345 | }; 346 | }; 347 | 348 | module.exports.reUploadPart = function (app) { 349 | return async function (req, res, next) { 350 | const { vodId, part, type } = req.body; 351 | if (!vodId) return res.status(400).json({ error: true, msg: "No vod id" }); 352 | if (!part) return res.status(400).json({ error: true, msg: "No part" }); 353 | if (!type) return res.status(400).json({ error: true, msg: "No type" }); 354 | 355 | res.status(200).json({ 356 | error: false, 357 | msg: `Reuploading ${vodId} Vod Part ${part}`, 358 | }); 359 | 360 | const vod_data = await app 361 | .service("vods") 362 | .get(vodId) 363 | .then((data) => data) 364 | .catch(() => null); 365 | 366 | let videoPath = 367 | type === "live" 368 | ? `${config.livePath}/${config.twitch.username}/${vod_data.stream_id}/${vod_data.stream_id}.mp4` 369 | : `${config.vodPath}/${vodId}.mp4`; 370 | 371 | if (!(await fileExists(videoPath))) { 372 | if (config.drive.upload) { 373 | videoPath = await drive.download(vodId, type, app); 374 | } else if (type === "vod") { 375 | videoPath = await vod.mp4Download(vodId); 376 | } else { 377 | videoPath = null; 378 | } 379 | } 380 | 381 | if (!videoPath) 382 | return console.error(`Could not find a download source for ${vodId}`); 383 | 384 | await vod.liveUploadPart( 385 | app, 386 | vodId, 387 | videoPath, 388 | config.youtube.splitDuration * parseInt(part) - 1, 389 | config.youtube.splitDuration, 390 | part, 391 | type 392 | ); 393 | }; 394 | }; 395 | 396 | module.exports.saveChapters = function (app) { 397 | return async function (req, res, next) { 398 | const { vodId, platform } = req.body; 399 | if (!vodId) return res.status(400).json({ error: true, msg: "No vod id" }); 400 | if (!platform) 401 | return res.status(400).json({ error: true, msg: "No platform" }); 402 | 403 | if (platform === "twitch") { 404 | const vodData = await twitch.getVodData(vodId); 405 | if (!vodData) 406 | return res.status(500).json({ 407 | error: true, 408 | msg: `Failed to get vod data for ${vodId}`, 409 | }); 410 | 411 | vod.saveChapters( 412 | vodData.id, 413 | app, 414 | dayjs.duration(`PT${vodData.duration.toUpperCase()}`).asSeconds() 415 | ); 416 | res 417 | .status(200) 418 | .json({ error: false, msg: `Saving Chapters for ${vodId}` }); 419 | } else if (platform === "kick") { 420 | //TODO 421 | res 422 | .status(200) 423 | .json({ error: false, msg: `Saving Chapters for ${vodId}` }); 424 | } else { 425 | res.status(400).json({ error: true, msg: `Platform not supported..` }); 426 | } 427 | }; 428 | }; 429 | 430 | module.exports.saveDuration = function (app) { 431 | return async function (req, res, next) { 432 | const { vodId, platform } = req.body; 433 | if (!vodId) return res.status(400).json({ error: true, msg: "No vod id" }); 434 | if (!platform) 435 | return res.status(400).json({ error: true, msg: "No platform" }); 436 | 437 | if (platform === "twitch") { 438 | const vodData = await twitch.getVodData(vodId); 439 | if (!vodData) 440 | return res.status(500).json({ 441 | error: true, 442 | msg: `Failed to get vod data for ${vodId}`, 443 | }); 444 | 445 | const exists = await app 446 | .service("vods") 447 | .get(vodId) 448 | .then(() => true) 449 | .catch(() => false); 450 | 451 | if (exists) { 452 | await app 453 | .service("vods") 454 | .patch(vodId, { 455 | duration: dayjs 456 | .duration(`PT${vodData.duration.toUpperCase()}`) 457 | .format("HH:mm:ss"), 458 | }) 459 | .then(() => 460 | res.status(200).json({ error: false, msg: "Saved duration!" }) 461 | ) 462 | .catch(() => 463 | res 464 | .status(500) 465 | .json({ error: true, msg: "Failed to save duration!" }) 466 | ); 467 | return; 468 | } 469 | } else if (platform === "kick") { 470 | //TODO 471 | return; 472 | } 473 | 474 | res.status(404).json({ error: true, msg: "Vod does not exist!" }); 475 | }; 476 | }; 477 | 478 | module.exports.addGame = function (app) { 479 | return async function (req, res, next) { 480 | const { 481 | vod_id, 482 | start_time, 483 | end_time, 484 | video_provider, 485 | video_id, 486 | game_id, 487 | game_name, 488 | thumbnail_url, 489 | } = req.body; 490 | if (vod_id == null) 491 | return res 492 | .status(400) 493 | .json({ error: true, msg: "Missing parameter vod_id" }); 494 | if (start_time == null) 495 | return res 496 | .status(400) 497 | .json({ error: true, msg: "Missing parameter: start_time" }); 498 | if (end_time == null) 499 | return res 500 | .status(400) 501 | .json({ error: true, msg: "Missing parameter: end_time" }); 502 | if (!video_provider) 503 | return res 504 | .status(400) 505 | .json({ error: true, msg: "Missing parameter: video_provider" }); 506 | if (!video_id) 507 | return res 508 | .status(400) 509 | .json({ error: true, msg: "Missing parameter: video_id" }); 510 | if (!game_id) 511 | return res 512 | .status(400) 513 | .json({ error: true, msg: "Missing parameter: game_id" }); 514 | if (!game_name) 515 | return res 516 | .status(400) 517 | .json({ error: true, msg: "Missing parameter: game_name" }); 518 | if (!thumbnail_url) 519 | return res 520 | .status(400) 521 | .json({ error: true, msg: "Missing parameter: thumbnail_url" }); 522 | 523 | const exists = await app 524 | .service("vods") 525 | .get(vod_id) 526 | .then(() => true) 527 | .catch(() => false); 528 | 529 | if (!exists) 530 | return res 531 | .status(400) 532 | .json({ error: true, msg: `${vod_id} does not exist!` }); 533 | 534 | await app 535 | .service("games") 536 | .create({ 537 | vodId: vod_id, 538 | start_time: start_time, 539 | end_time: end_time, 540 | video_provider: video_provider, 541 | video_id: video_id, 542 | game_id: game_id, 543 | game_name: game_name, 544 | thumbnail_url: thumbnail_url, 545 | }) 546 | .then(() => { 547 | console.info(`Created ${game_name} in games DB for ${vod_id}`); 548 | res.status(200).json({ 549 | error: false, 550 | msg: `Created ${game_name} in games DB for ${vod_id}`, 551 | }); 552 | }) 553 | .catch((e) => { 554 | console.error(e); 555 | res.status(500).json({ 556 | error: true, 557 | msg: `Failed to create ${game_name} in games DB for ${vod_id}`, 558 | }); 559 | }); 560 | }; 561 | }; 562 | 563 | module.exports.saveEmotes = function (app) { 564 | return async function (req, res, next) { 565 | const { vodId } = req.body; 566 | if (!vodId) return res.status(400).json({ error: true, msg: "No VodId" }); 567 | 568 | emotes.save(vodId, app); 569 | res.status(200).json({ error: false, msg: "Saving emotes.." }); 570 | }; 571 | }; 572 | 573 | module.exports.vodUpload = function (app) { 574 | return async function (req, res, next) { 575 | const { vodId, type } = req.body; 576 | if (!vodId) return res.status(400).json({ error: true, msg: "No vod id" }); 577 | if (!type) return res.status(400).json({ error: true, msg: "No type" }); 578 | 579 | res.status(200).json({ 580 | error: false, 581 | msg: `Reuploading ${vodId} Vod`, 582 | }); 583 | 584 | let videoPath = `${ 585 | type === "live" ? config.livePath : config.vodPath 586 | }/${vodId}.mp4`; 587 | 588 | if (!(await fileExists(videoPath))) { 589 | if (config.drive.upload) { 590 | videoPath = await drive.download(vodId, type, app); 591 | } else { 592 | if (vodData.platform === "twitch") { 593 | videoPath = await vod.mp4Download(vodId); 594 | } else if (vodData.platform === "kick") { 595 | videoPath = await kick.downloadMP4( 596 | app, 597 | config.kick.username, 598 | game.vodId 599 | ); 600 | } 601 | } 602 | } 603 | 604 | if (!videoPath) 605 | return console.error( 606 | `Could not find a download source for ${req.body.vodId}` 607 | ); 608 | 609 | vod.manualVodUpload(app, vodId, videoPath, type); 610 | }; 611 | }; 612 | 613 | module.exports.gameUpload = function (app) { 614 | return async function (req, res, next) { 615 | const { vodId, type, chapterIndex } = req.body; 616 | if (!vodId) return res.status(400).json({ error: true, msg: "No vod id" }); 617 | if (!type) return res.status(400).json({ error: true, msg: "No type" }); 618 | if (chapterIndex == null) 619 | return res.status(400).json({ error: true, msg: "No chapter" }); 620 | 621 | let vodData; 622 | await app 623 | .service("vods") 624 | .get(vodId) 625 | .then((data) => { 626 | vodData = data; 627 | }) 628 | .catch(() => {}); 629 | 630 | if (!vodData) 631 | return res.status(404).json({ 632 | error: true, 633 | msg: "Vod does not exist", 634 | }); 635 | 636 | const game = vodData.chapters[chapterIndex]; 637 | if (!game) 638 | return res.status(404).json({ 639 | error: true, 640 | msg: "Chapter does not exist", 641 | }); 642 | 643 | res.status(200).json({ 644 | error: false, 645 | msg: `Uploading ${chapter.name} from ${vodId} Vod`, 646 | }); 647 | 648 | let videoPath = `${ 649 | type === "live" ? config.livePath : config.vodPath 650 | }/${vodId}.mp4`; 651 | 652 | if (!(await fileExists(videoPath))) { 653 | if (config.drive.upload) { 654 | videoPath = await drive.download(game.vodId, type, app); 655 | } else { 656 | if (vodData.platform === "twitch") { 657 | videoPath = await vod.mp4Download(game.vodId); 658 | } else if (vodData.platform === "kick") { 659 | videoPath = await kick.downloadMP4( 660 | app, 661 | config.kick.username, 662 | game.vodId 663 | ); 664 | } 665 | } 666 | } 667 | 668 | if (!videoPath) 669 | return console.error( 670 | `Could not find a download source for ${game.vodId}` 671 | ); 672 | 673 | vod.manualGameUpload( 674 | app, 675 | vodData, 676 | { 677 | gameId: null, 678 | vodId: vodId, 679 | date: vodData.createdAt, 680 | chapter: game, 681 | }, 682 | videoPath 683 | ); 684 | }; 685 | }; 686 | 687 | module.exports.reuploadGame = function (app) { 688 | return async function (req, res, next) { 689 | const { gameId, type } = req.body; 690 | if (!gameId) 691 | return res.status(400).json({ error: true, msg: "No game id" }); 692 | if (!type) return res.status(400).json({ error: true, msg: "No type" }); 693 | 694 | let game; 695 | await app 696 | .service("games") 697 | .get(gameId) 698 | .then((data) => { 699 | game = data; 700 | }) 701 | .catch(() => {}); 702 | 703 | if (!game) 704 | return res.status(404).json({ 705 | error: true, 706 | msg: "Game does not exist", 707 | }); 708 | 709 | let vodData; 710 | await app 711 | .service("vods") 712 | .get(game.vodId) 713 | .then((data) => { 714 | vodData = data; 715 | }) 716 | .catch(() => {}); 717 | 718 | if (!vodData) 719 | return res.status(404).json({ 720 | error: true, 721 | msg: "Vod does not exist", 722 | }); 723 | 724 | res.status(200).json({ 725 | error: false, 726 | msg: `Uploading ${game.game_name} from ${game.vodId} Vod`, 727 | }); 728 | 729 | let videoPath = `${type === "live" ? config.livePath : config.vodPath}/${ 730 | game.vodId 731 | }.mp4`; 732 | 733 | if (!(await fileExists(videoPath))) { 734 | if (config.drive.upload) { 735 | videoPath = await drive.download(game.vodId, type, app); 736 | } else { 737 | if (vodData.platform === "twitch") { 738 | videoPath = await vod.mp4Download(game.vodId); 739 | } else if (vodData.platform === "kick") { 740 | videoPath = await kick.downloadMP4( 741 | app, 742 | config.kick.username, 743 | game.vodId 744 | ); 745 | } 746 | } 747 | } 748 | 749 | if (!videoPath) 750 | return console.error( 751 | `Could not find a download source for ${game.vodId}` 752 | ); 753 | 754 | vod.manualGameUpload( 755 | app, 756 | vodData, 757 | { 758 | gameId: game.id, 759 | title: game.title, 760 | vodId: game.vodId, 761 | date: vodData.createdAt, 762 | chapter: { 763 | end: game.end_time, 764 | start: game.start_time, 765 | name: game.game_name, 766 | }, 767 | }, 768 | videoPath 769 | ); 770 | }; 771 | }; 772 | 773 | const fileExists = async (file) => { 774 | return fs.promises 775 | .access(file, fs.constants.F_OK) 776 | .then(() => true) 777 | .catch(() => false); 778 | }; 779 | -------------------------------------------------------------------------------- /src/middleware/dmca.js: -------------------------------------------------------------------------------- 1 | const config = require("../../config/config.json"); 2 | const ffmpeg = require("fluent-ffmpeg"); 3 | const drive = require("./drive"); 4 | const youtube = require("./youtube"); 5 | const vod = require("./vod"); 6 | const kick = require("./kick"); 7 | const fs = require("fs"); 8 | const path = require("path"); 9 | const readline = require("readline"); 10 | const dayjs = require("dayjs"); 11 | const utc = require("dayjs/plugin/utc"); 12 | const timezone = require("dayjs/plugin/timezone"); 13 | dayjs.extend(utc); 14 | dayjs.extend(timezone); 15 | const { getDuration } = require("./ffmpeg"); 16 | 17 | const mute = async (vodPath, muteSection, vodId) => { 18 | let returnPath; 19 | await new Promise((resolve, reject) => { 20 | const filePath = path.normalize( 21 | `${path.dirname(vodPath)}/${vodId}-muted.mp4` 22 | ); 23 | const ffmpeg_process = ffmpeg(vodPath); 24 | ffmpeg_process 25 | .videoCodec("copy") 26 | .audioFilters(muteSection) 27 | .toFormat("mp4") 28 | .on("progress", (progress) => { 29 | if ((process.env.NODE_ENV || "").trim() !== "production") { 30 | readline.clearLine(process.stdout, 0); 31 | readline.cursorTo(process.stdout, 0, null); 32 | process.stdout.write( 33 | `MUTE VIDEO PROGRESS: ${Math.round(progress.percent)}%` 34 | ); 35 | } 36 | }) 37 | .on("start", (cmd) => { 38 | if ((process.env.NODE_ENV || "").trim() !== "production") { 39 | console.info(cmd); 40 | } 41 | }) 42 | .on("error", function (err) { 43 | ffmpeg_process.kill("SIGKILL"); 44 | reject(err); 45 | }) 46 | .on("end", function () { 47 | resolve(filePath); 48 | }) 49 | .saveToFile(filePath); 50 | }) 51 | .then((result) => { 52 | returnPath = result; 53 | console.info("\n"); 54 | }) 55 | .catch((e) => { 56 | console.error("\nffmpeg error occurred: " + e); 57 | }); 58 | return returnPath; 59 | }; 60 | 61 | const blackoutVideo = async (vodPath, vodId, start, duration, end) => { 62 | const start_video_path = await getStartVideo(vodPath, vodId, start); 63 | if (!start_video_path) { 64 | console.error("failed to get start video"); 65 | return null; 66 | } 67 | const clip_path = await getClip(vodPath, vodId, start, duration); 68 | if (!clip_path) { 69 | console.error("failed to get clip"); 70 | return null; 71 | } 72 | const trim_clip_path = await getTrimmedClip(clip_path, vodId); 73 | if (!trim_clip_path) { 74 | console.error("failed to get trimmed clip"); 75 | return null; 76 | } 77 | const end_video_path = await getEndVideo(vodPath, vodId, end); 78 | if (!end_video_path) { 79 | console.error("failed to get end video"); 80 | return null; 81 | } 82 | const list = await getTextList( 83 | vodId, 84 | start_video_path, 85 | trim_clip_path, 86 | end_video_path, 87 | vodPath 88 | ); 89 | if (!list) { 90 | console.error("failed to get text list"); 91 | return null; 92 | } 93 | const returnPath = await concat(vodId, list); 94 | if (!returnPath) { 95 | console.error("failed to concat"); 96 | return null; 97 | } 98 | fs.unlinkSync(start_video_path); 99 | fs.unlinkSync(trim_clip_path); 100 | fs.unlinkSync(end_video_path); 101 | fs.unlinkSync(list); 102 | fs.unlinkSync(clip_path); 103 | return returnPath; 104 | }; 105 | 106 | const getTextList = async ( 107 | vodId, 108 | start_video_path, 109 | trim_clip_path, 110 | end_video_path, 111 | vodPath 112 | ) => { 113 | const textPath = path.normalize(`${path.dirname(vodPath)}/${vodId}-list.txt`); 114 | fs.writeFileSync( 115 | textPath, 116 | `file '${start_video_path}'\nfile '${trim_clip_path}'\nfile '${end_video_path}'` 117 | ); 118 | return textPath; 119 | }; 120 | 121 | const concat = async (vodId, list) => { 122 | let returnPath; 123 | await new Promise((resolve, reject) => { 124 | const filePath = path.normalize( 125 | `${path.dirname(list)}/${vodId}-trimmed.mp4` 126 | ); 127 | const ffmpeg_process = ffmpeg(list); 128 | ffmpeg_process 129 | .inputOptions(["-f concat", "-safe 0"]) 130 | .videoCodec("copy") 131 | .audioCodec("copy") 132 | .on("progress", (progress) => { 133 | if ((process.env.NODE_ENV || "").trim() !== "production") { 134 | readline.clearLine(process.stdout, 0); 135 | readline.cursorTo(process.stdout, 0, null); 136 | process.stdout.write( 137 | `CONCAT PROGRESS: ${Math.round(progress.percent)}%` 138 | ); 139 | } 140 | }) 141 | .on("start", (cmd) => { 142 | if ((process.env.NODE_ENV || "").trim() !== "production") { 143 | console.info(cmd); 144 | } 145 | }) 146 | .on("error", function (err) { 147 | ffmpeg_process.kill("SIGKILL"); 148 | reject(err); 149 | }) 150 | .on("end", function () { 151 | resolve(filePath); 152 | }) 153 | .saveToFile(filePath); 154 | }) 155 | .then((result) => { 156 | returnPath = result; 157 | console.info("\n"); 158 | }) 159 | .catch((e) => { 160 | console.error("\nffmpeg error occurred: " + e); 161 | }); 162 | return returnPath; 163 | }; 164 | 165 | const getStartVideo = async (vodPath, vodId, start) => { 166 | let returnPath; 167 | await new Promise((resolve, reject) => { 168 | const filePath = path.normalize( 169 | `${path.dirname(vodPath)}/${vodId}-start.mp4` 170 | ); 171 | const ffmpeg_process = ffmpeg(vodPath); 172 | ffmpeg_process 173 | .videoCodec("copy") 174 | .audioCodec("copy") 175 | .duration(start) 176 | .toFormat("mp4") 177 | .on("progress", (progress) => { 178 | if ((process.env.NODE_ENV || "").trim() !== "production") { 179 | readline.clearLine(process.stdout, 0); 180 | readline.cursorTo(process.stdout, 0, null); 181 | process.stdout.write( 182 | `GET START VIDEO PROGRESS: ${Math.round(progress.percent)}%` 183 | ); 184 | } 185 | }) 186 | .on("start", (cmd) => { 187 | if ((process.env.NODE_ENV || "").trim() !== "production") { 188 | console.info(cmd); 189 | } 190 | }) 191 | .on("error", function (err) { 192 | ffmpeg_process.kill("SIGKILL"); 193 | reject(err); 194 | }) 195 | .on("end", function () { 196 | resolve(filePath); 197 | }) 198 | .saveToFile(filePath); 199 | }) 200 | .then((result) => { 201 | returnPath = result; 202 | console.info("\n"); 203 | }) 204 | .catch((e) => { 205 | console.error("\nffmpeg error occurred: " + e); 206 | }); 207 | return returnPath; 208 | }; 209 | 210 | const getClip = async (vodPath, vodId, start, duration) => { 211 | let returnPath; 212 | await new Promise((resolve, reject) => { 213 | const filePath = path.normalize( 214 | `${path.dirname(vodPath)}/${vodId}-clip.mp4` 215 | ); 216 | const ffmpeg_process = ffmpeg(vodPath); 217 | ffmpeg_process 218 | .videoCodec("copy") 219 | .audioCodec("copy") 220 | .seekOutput(start) 221 | .duration(duration) 222 | .toFormat("mp4") 223 | .on("progress", (progress) => { 224 | if ((process.env.NODE_ENV || "").trim() !== "production") { 225 | readline.clearLine(process.stdout, 0); 226 | readline.cursorTo(process.stdout, 0, null); 227 | process.stdout.write( 228 | `GET CLIP PROGRESS: ${Math.round(progress.percent)}%` 229 | ); 230 | } 231 | }) 232 | .on("start", (cmd) => { 233 | if ((process.env.NODE_ENV || "").trim() !== "production") { 234 | console.info(cmd); 235 | } 236 | }) 237 | .on("error", function (err) { 238 | ffmpeg_process.kill("SIGKILL"); 239 | reject(err); 240 | }) 241 | .on("end", function () { 242 | resolve(filePath); 243 | }) 244 | .saveToFile(filePath); 245 | }) 246 | .then((result) => { 247 | returnPath = result; 248 | console.info("\n"); 249 | }) 250 | .catch((e) => { 251 | console.error("\nffmpeg error occurred: " + e); 252 | }); 253 | return returnPath; 254 | }; 255 | 256 | const getTrimmedClip = async (clipPath, vodId) => { 257 | let returnPath; 258 | await new Promise((resolve, reject) => { 259 | const filePath = path.normalize( 260 | `${path.dirname(clipPath)}/${vodId}-clip-muted.mp4` 261 | ); 262 | const ffmpeg_process = ffmpeg(clipPath); 263 | ffmpeg_process 264 | .audioCodec("copy") 265 | .videoFilter("geq=0:128:128") 266 | .toFormat("mp4") 267 | .on("progress", (progress) => { 268 | if ((process.env.NODE_ENV || "").trim() !== "production") { 269 | readline.clearLine(process.stdout, 0); 270 | readline.cursorTo(process.stdout, 0, null); 271 | process.stdout.write( 272 | `GET TRIMMED CLIP PROGRESS: ${Math.round(progress.percent)}%` 273 | ); 274 | } 275 | }) 276 | .on("start", (cmd) => { 277 | if ((process.env.NODE_ENV || "").trim() !== "production") { 278 | console.info(cmd); 279 | } 280 | }) 281 | .on("error", function (err) { 282 | ffmpeg_process.kill("SIGKILL"); 283 | reject(err); 284 | }) 285 | .on("end", function () { 286 | resolve(filePath); 287 | }) 288 | .saveToFile(filePath); 289 | }) 290 | .then((result) => { 291 | returnPath = result; 292 | console.info("\n"); 293 | }) 294 | .catch((e) => { 295 | console.error("\nffmpeg error occurred: " + e); 296 | }); 297 | return returnPath; 298 | }; 299 | 300 | const getEndVideo = async (vodPath, vodId, end) => { 301 | let returnPath; 302 | await new Promise((resolve, reject) => { 303 | const filePath = path.normalize( 304 | `${path.dirname(vodPath)}/${vodId}-end.mp4` 305 | ); 306 | const ffmpeg_process = ffmpeg(vodPath); 307 | ffmpeg_process 308 | .videoCodec("copy") 309 | .audioCodec("copy") 310 | .seekOutput(end) 311 | .toFormat("mp4") 312 | .on("progress", (progress) => { 313 | if ((process.env.NODE_ENV || "").trim() !== "production") { 314 | readline.clearLine(process.stdout, 0); 315 | readline.cursorTo(process.stdout, 0, null); 316 | process.stdout.write( 317 | `GET END VIDEO PROGRESS: ${Math.round(progress.percent)}%` 318 | ); 319 | } 320 | }) 321 | .on("start", (cmd) => { 322 | if ((process.env.NODE_ENV || "").trim() !== "production") { 323 | console.info(cmd); 324 | } 325 | }) 326 | .on("error", function (err) { 327 | ffmpeg_process.kill("SIGKILL"); 328 | reject(err); 329 | }) 330 | .on("end", function () { 331 | resolve(filePath); 332 | }) 333 | .saveToFile(filePath); 334 | }) 335 | .then((result) => { 336 | returnPath = result; 337 | console.info("\n"); 338 | }) 339 | .catch((e) => { 340 | console.error("\nffmpeg error occurred: " + e); 341 | }); 342 | return returnPath; 343 | }; 344 | 345 | module.exports = function (app) { 346 | return async function (req, res, next) { 347 | const { vodId, type, receivedClaims, platform } = req.body; 348 | 349 | if (!receivedClaims) 350 | return res.status(400).json({ error: true, msg: "No claims" }); 351 | if (!vodId) return res.status(400).json({ error: true, msg: "No vod id" }); 352 | if (!type) return res.status(400).json({ error: true, msg: "No type" }); 353 | if (!platform) 354 | return res.status(400).json({ error: true, msg: "No platform" }); 355 | 356 | let vod_data; 357 | await app 358 | .service("vods") 359 | .get(vodId) 360 | .then((data) => { 361 | vod_data = data; 362 | }) 363 | .catch(() => {}); 364 | 365 | if (!vod_data) 366 | return console.error("Failed to download video: no VOD in database"); 367 | 368 | res.status(200).json({ 369 | error: false, 370 | msg: `Muting the DMCA content for ${vodId}...`, 371 | }); 372 | 373 | let videoPath = 374 | type === "live" 375 | ? `${config.livePath}/${config.twitch.username}/${vod_data.stream_id}/${vod_data.stream_id}.mp4` 376 | : `${config.vodPath}/${vodId}.mp4`; 377 | 378 | if (!(await fileExists(videoPath))) { 379 | if (config.drive.upload) { 380 | videoPath = await drive.download(vodId, type, app); 381 | } else if (type === "vod") { 382 | if (platform === "twitch") { 383 | vodPath = await vod.mp4Download(vodId); 384 | } else if (platform === "kick") { 385 | vodPath = await kick.download(vodId); 386 | } 387 | } else { 388 | videoPath = null; 389 | } 390 | } 391 | 392 | if (!videoPath) 393 | return console.error(`Could not find a download source for ${vodId}`); 394 | 395 | let muteSection = [], 396 | newVodPath, 397 | blackoutPath; 398 | for (let dmca of receivedClaims) { 399 | const policyType = dmca.claimPolicy.primaryPolicy.policyType; 400 | if ( 401 | policyType === "POLICY_TYPE_GLOBAL_BLOCK" || 402 | policyType === "POLICY_TYPE_MOSTLY_GLOBAL_BLOCK" || 403 | policyType === "POLICY_TYPE_BLOCK" 404 | ) { 405 | if (dmca.type === "CLAIM_TYPE_AUDIO") { 406 | muteSection.push( 407 | `volume=0:enable='between(t,${ 408 | dmca.matchDetails.longestMatchStartTimeSeconds 409 | },${ 410 | parseInt(dmca.matchDetails.longestMatchDurationSeconds) + 411 | parseInt(dmca.matchDetails.longestMatchStartTimeSeconds) 412 | })'` 413 | ); 414 | } else if (dmca.type === "CLAIM_TYPE_VISUAL") { 415 | console.info( 416 | `Trying to blackout ${ 417 | blackoutPath ? blackoutPath : videoPath 418 | }. Claim: ${JSON.stringify(dmca.asset.metadata)}` 419 | ); 420 | blackoutPath = await blackoutVideo( 421 | blackoutPath ? blackoutPath : videoPath, 422 | vodId, 423 | dmca.matchDetails.longestMatchStartTimeSeconds, 424 | dmca.matchDetails.longestMatchDurationSeconds, 425 | parseInt(dmca.matchDetails.longestMatchStartTimeSeconds) + 426 | parseInt(dmca.matchDetails.longestMatchDurationSeconds) 427 | ); 428 | } else if (dmca.type === "CLAIM_TYPE_AUDIOVISUAL") { 429 | muteSection.push( 430 | `volume=0:enable='between(t,${ 431 | dmca.matchDetails.longestMatchStartTimeSeconds 432 | },${ 433 | parseInt(dmca.matchDetails.longestMatchDurationSeconds) + 434 | parseInt(dmca.matchDetails.longestMatchStartTimeSeconds) 435 | })'` 436 | ); 437 | console.info( 438 | `Trying to blackout ${ 439 | blackoutPath ? blackoutPath : videoPath 440 | }. Claim: ${JSON.stringify(dmca.asset.metadata)}` 441 | ); 442 | blackoutPath = await blackoutVideo( 443 | blackoutPath ? blackoutPath : videoPath, 444 | vodId, 445 | dmca.matchDetails.longestMatchStartTimeSeconds, 446 | dmca.matchDetails.longestMatchDurationSeconds, 447 | parseInt(dmca.matchDetails.longestMatchStartTimeSeconds) + 448 | parseInt(dmca.matchDetails.longestMatchDurationSeconds) 449 | ); 450 | } 451 | } 452 | } 453 | 454 | if (muteSection.length > 0) { 455 | console.info(`Trying to mute ${blackoutPath ? blackoutPath : videoPath}`); 456 | newVodPath = await mute( 457 | blackoutPath ? blackoutPath : videoPath, 458 | muteSection, 459 | vodId 460 | ); 461 | if (!newVodPath) return console.error("failed to mute video"); 462 | } 463 | 464 | vod.upload(vodId, app, newVodPath, type, platform); 465 | }; 466 | }; 467 | 468 | module.exports.part = function (app) { 469 | return async function (req, res, next) { 470 | const { vodId, part, type, receivedClaims } = req.body; 471 | 472 | if (!receivedClaims) 473 | return res.status(400).json({ error: true, msg: "No claims" }); 474 | if (!vodId) return res.status(400).json({ error: true, msg: "No vod id" }); 475 | if (!part) return res.status(400).json({ error: true, msg: "No part" }); 476 | if (!type) return res.status(400).json({ error: true, msg: "No type" }); 477 | 478 | res.status(200).json({ 479 | error: false, 480 | msg: `Trimming DMCA Content from ${vodId} Vod Part ${part}`, 481 | }); 482 | 483 | let vod_data; 484 | await app 485 | .service("vods") 486 | .get(vodId) 487 | .then((data) => { 488 | vod_data = data; 489 | }) 490 | .catch(() => {}); 491 | 492 | if (!vod_data) return console.error("Failed get vod: no VOD in database"); 493 | 494 | let videoPath = 495 | type === "live" 496 | ? `${config.livePath}/${config.twitch.username}/${vod_data.stream_id}/${vod_data.stream_id}.mp4` 497 | : `${config.vodPath}/${vodId}.mp4`; 498 | 499 | if (!(await fileExists(videoPath))) { 500 | if (config.drive.upload) { 501 | videoPath = await drive.download(vodId, type, app); 502 | } else if (type === "vod") { 503 | videoPath = await vod.mp4Download(vodId); 504 | } else { 505 | videoPath = null; 506 | } 507 | } 508 | 509 | if (!videoPath) 510 | return console.error(`Could not find a download source for ${vodId}`); 511 | 512 | const trimmedPath = await vod.trim( 513 | videoPath, 514 | vodId, 515 | config.youtube.splitDuration * (parseInt(part) - 1), 516 | config.youtube.splitDuration 517 | ); 518 | 519 | if (!trimmedPath) 520 | return console.error(`Failed Trim for ${vodId} Part ${part}`); 521 | 522 | console.info("Finished Trim.."); 523 | 524 | let muteSection = [], 525 | newVodPath, 526 | blackoutPath; 527 | for (let dmca of receivedClaims) { 528 | const policyType = dmca.claimPolicy.primaryPolicy.policyType; 529 | if ( 530 | policyType === "POLICY_TYPE_GLOBAL_BLOCK" || 531 | policyType === "POLICY_TYPE_MOSTLY_GLOBAL_BLOCK" || 532 | policyType === "POLICY_TYPE_BLOCK" 533 | ) { 534 | if (dmca.type === "CLAIM_TYPE_AUDIO") { 535 | muteSection.push( 536 | `volume=0:enable='between(t,${ 537 | dmca.matchDetails.longestMatchStartTimeSeconds 538 | },${ 539 | parseInt(dmca.matchDetails.longestMatchDurationSeconds) + 540 | parseInt(dmca.matchDetails.longestMatchStartTimeSeconds) 541 | })'` 542 | ); 543 | } else if (dmca.type === "CLAIM_TYPE_VISUAL") { 544 | console.info( 545 | `Trying to blackout ${ 546 | blackoutPath ? blackoutPath : trimmedPath 547 | }. Claim: ${JSON.stringify(dmca.asset.metadata)}` 548 | ); 549 | blackoutPath = await blackoutVideo( 550 | blackoutPath ? blackoutPath : trimmedPath, 551 | vodId, 552 | dmca.matchDetails.longestMatchStartTimeSeconds, 553 | dmca.matchDetails.longestMatchDurationSeconds, 554 | parseInt(dmca.matchDetails.longestMatchStartTimeSeconds) + 555 | parseInt(dmca.matchDetails.longestMatchDurationSeconds) 556 | ); 557 | } else if (dmca.type === "CLAIM_TYPE_AUDIOVISUAL") { 558 | muteSection.push( 559 | `volume=0:enable='between(t,${ 560 | dmca.matchDetails.longestMatchStartTimeSeconds 561 | },${ 562 | parseInt(dmca.matchDetails.longestMatchDurationSeconds) + 563 | parseInt(dmca.matchDetails.longestMatchStartTimeSeconds) 564 | })'` 565 | ); 566 | console.info( 567 | `Trying to blackout ${ 568 | blackoutPath ? blackoutPath : trimmedPath 569 | }. Claim: ${JSON.stringify(dmca.asset.metadata)}` 570 | ); 571 | blackoutPath = await blackoutVideo( 572 | blackoutPath ? blackoutPath : trimmedPath, 573 | vodId, 574 | dmca.matchDetails.longestMatchStartTimeSeconds, 575 | dmca.matchDetails.longestMatchDurationSeconds, 576 | parseInt(dmca.matchDetails.longestMatchStartTimeSeconds) + 577 | parseInt(dmca.matchDetails.longestMatchDurationSeconds) 578 | ); 579 | } 580 | } 581 | } 582 | 583 | if (muteSection.length > 0) { 584 | console.info( 585 | `Trying to mute ${blackoutPath ? blackoutPath : trimmedPath}` 586 | ); 587 | newVodPath = await mute( 588 | blackoutPath ? blackoutPath : trimmedPath, 589 | muteSection, 590 | vodId 591 | ); 592 | if (!newVodPath) return console.error("failed to mute video"); 593 | 594 | fs.unlinkSync(trimmedPath); 595 | } 596 | 597 | if (!newVodPath && !blackoutPath) 598 | return console.error( 599 | "nothing to mute or blackout. don't try to upload.." 600 | ); 601 | 602 | const duration = await getDuration(newVodPath ? newVodPath : blackoutPath); 603 | 604 | await youtube.upload( 605 | { 606 | path: newVodPath ? newVodPath : blackoutPath, 607 | title: 608 | type === "vod" 609 | ? `${config.channel} VOD - ${dayjs(vod_data.createdAt) 610 | .tz(config.timezone) 611 | .format("MMMM DD YYYY") 612 | .toUpperCase()} Part ${part}` 613 | : `${config.channel} Live VOD - ${dayjs(vod_data.createdAt) 614 | .tz(config.timezone) 615 | .format("MMMM DD YYYY") 616 | .toUpperCase()} Part ${part}`, 617 | public: 618 | config.youtube.multiTrack && type === "live" 619 | ? true 620 | : !config.youtube.multiTrack && type === "vod" 621 | ? true 622 | : false, 623 | vod: vod_data, 624 | part: part, 625 | type: type, 626 | duration: duration, 627 | }, 628 | app 629 | ); 630 | 631 | if (blackoutPath) fs.unlinkSync(blackoutPath); 632 | }; 633 | }; 634 | 635 | const fileExists = async (file) => { 636 | return fs.promises 637 | .access(file, fs.constants.F_OK) 638 | .then(() => true) 639 | .catch(() => false); 640 | }; -------------------------------------------------------------------------------- /src/middleware/drive.js: -------------------------------------------------------------------------------- 1 | const config = require("../../config/config.json"); 2 | const fs = require("fs"); 3 | const readline = require("readline"); 4 | const path = require("path"); 5 | const { google } = require("googleapis"); 6 | 7 | module.exports.upload = async (vodId, path, app, type = "vod") => { 8 | const oauth2Client = app.get("driveOauth2Client"); 9 | const drive = google.drive({ 10 | version: "v3", 11 | auth: oauth2Client, 12 | }); 13 | await drive.files.list(); 14 | await sleep(5000); 15 | 16 | const fileSize = fs.statSync(path).size; 17 | const res = await drive.files.create( 18 | { 19 | auth: oauth2Client, 20 | resource: { 21 | name: `${vodId}_${type}.mp4`, 22 | parents: config.drive.parents, 23 | }, 24 | media: { 25 | body: fs.createReadStream(path), 26 | }, 27 | }, 28 | { 29 | onUploadProgress: (evt) => { 30 | if ((process.env.NODE_ENV || "").trim() !== "production") { 31 | const progress = (evt.bytesRead / fileSize) * 100; 32 | readline.clearLine(process.stdout, 0); 33 | readline.cursorTo(process.stdout, 0, null); 34 | process.stdout.write(`DRIVE UPLOAD PROGRESS: ${Math.round(progress)}%`); 35 | } 36 | }, 37 | } 38 | ); 39 | console.log("\n\n"); 40 | console.log(res.data); 41 | 42 | let vod_data; 43 | await app 44 | .service("vods") 45 | .get(vodId) 46 | .then((data) => { 47 | vod_data = data; 48 | }) 49 | .catch(() => {}); 50 | 51 | if (!vod_data) return console.error("Failed to upload to drive: no VOD in database"); 52 | 53 | vod_data.drive.push({ 54 | id: res.data.id, 55 | type: type, 56 | }); 57 | 58 | await app 59 | .service("vods") 60 | .patch(vod_data.id, { 61 | drive: vod_data.drive, 62 | }) 63 | .catch((e) => { 64 | console.error(e); 65 | }); 66 | }; 67 | 68 | const sleep = (ms) => { 69 | return new Promise((resolve) => setTimeout(resolve, ms)); 70 | }; 71 | 72 | module.exports.download = async (vodId, type, app) => { 73 | let vod; 74 | await app 75 | .service("vods") 76 | .get(vodId) 77 | .then((data) => { 78 | vod = data; 79 | }) 80 | .catch(() => {}); 81 | 82 | if (!vod) return console.error("Failed to download from drive: no VOD in database"); 83 | 84 | let driveId; 85 | 86 | for (let drive of vod.drive) { 87 | if (type !== drive.type) continue; 88 | driveId = drive.id; 89 | } 90 | 91 | if (!driveId) return console.error("Failed to download from drive: no DRIVE ID in database"); 92 | 93 | console.info(`Drive Download: ${driveId} for ${type} ${vodId}`); 94 | const oauth2Client = app.get("driveOauth2Client"); 95 | const drive = google.drive({ 96 | version: "v3", 97 | auth: oauth2Client, 98 | }); 99 | await drive.files.list(); 100 | await sleep(5000); 101 | 102 | const filePath = path.join(type === "vod" ? config.vodPath : config.livePath, `${vodId}.mp4`); 103 | 104 | await drive.files.get({ fileId: driveId, alt: "media" }, { responseType: "stream" }).then((res) => { 105 | return new Promise((resolve, reject) => { 106 | const dest = fs.createWriteStream(filePath); 107 | let progress = 0; 108 | 109 | res.data 110 | .on("end", () => { 111 | resolve(filePath); 112 | }) 113 | .on("error", (err) => { 114 | console.error("Error downloading file."); 115 | reject(err); 116 | }) 117 | .on("data", (d) => { 118 | progress += d.length; 119 | if (process.stdout.isTTY && (process.env.NODE_ENV || "").trim() !== "production") { 120 | process.stdout.clearLine(); 121 | process.stdout.cursorTo(0); 122 | process.stdout.write(`Downloaded ${progress} bytes`); 123 | } 124 | }) 125 | .pipe(dest); 126 | }); 127 | }); 128 | 129 | return filePath; 130 | }; 131 | -------------------------------------------------------------------------------- /src/middleware/emotes.js: -------------------------------------------------------------------------------- 1 | const config = require("../../config/config.json"); 2 | const axios = require("axios"); 3 | 4 | module.exports.save = async (vodId, app) => { 5 | const BASE_FFZ_EMOTE_API = "https://api.frankerfacez.com/v1", 6 | BASE_BTTV_EMOTE_API = "https://api.betterttv.net/3", 7 | BASE_7TV_EMOTE_API = "https://7tv.io/v3"; 8 | 9 | const twitchId = config.twitch.id; 10 | 11 | const FFZ_EMOTES = await axios(`${BASE_FFZ_EMOTE_API}/room/id/${twitchId}`, { 12 | method: "GET", 13 | }) 14 | .then((response) => { 15 | const emotes = response.data.sets[response.data.room.set].emoticons; 16 | let newEmotes = []; 17 | for (let emote of emotes) { 18 | newEmotes.push({ id: emote.id, code: emote.name }); 19 | } 20 | return newEmotes; 21 | }) 22 | .catch(async (e) => { 23 | console.error(e.response ? e.response.data : e); 24 | return null; 25 | }); 26 | 27 | let BTTV_EMOTES = await axios(`${BASE_BTTV_EMOTE_API}/cached/emotes/global`, { 28 | method: "GET", 29 | }) 30 | .then((response) => { 31 | const emotes = response.data; 32 | let newEmotes = []; 33 | for (let emote of emotes) { 34 | newEmotes.push({ id: emote.id, code: emote.code }); 35 | } 36 | return newEmotes; 37 | }) 38 | .catch(async (e) => { 39 | console.error(e.response ? e.response.data : e); 40 | return null; 41 | }); 42 | 43 | const BTTV_CHANNEL_EMOTES = await axios( 44 | `${BASE_BTTV_EMOTE_API}/cached/users/twitch/${twitchId}`, 45 | { 46 | method: "GET", 47 | } 48 | ) 49 | .then((response) => { 50 | const emotes = response.data.channelEmotes.concat( 51 | response.data.sharedEmotes 52 | ); 53 | let newEmotes = []; 54 | for (let emote of emotes) { 55 | newEmotes.push({ id: emote.id, code: emote.code }); 56 | } 57 | return newEmotes; 58 | }) 59 | .catch(async (e) => { 60 | console.error(e.response ? e.response.data : e); 61 | return null; 62 | }); 63 | 64 | if (BTTV_CHANNEL_EMOTES) 65 | BTTV_EMOTES = BTTV_EMOTES.concat(BTTV_CHANNEL_EMOTES); 66 | 67 | const _7TV_EMOTES = await axios( 68 | `${BASE_7TV_EMOTE_API}/users/twitch/${twitchId}`, 69 | { 70 | method: "GET", 71 | } 72 | ) 73 | .then((response) => { 74 | const emotes = response.data.emote_set.emotes; 75 | let newEmotes = []; 76 | for (let emote of emotes) { 77 | newEmotes.push({ id: emote.id, code: emote.name }); 78 | } 79 | return newEmotes; 80 | }) 81 | .catch(async (e) => { 82 | console.error(e.response ? e.response.data : e); 83 | return null; 84 | }); 85 | 86 | const exists = await app 87 | .service("emotes") 88 | .get(vodId) 89 | .then(() => true) 90 | .catch(() => false); 91 | 92 | if (!exists) 93 | await app 94 | .service("emotes") 95 | .create({ 96 | vodId: vodId, 97 | ffz_emotes: FFZ_EMOTES ? FFZ_EMOTES : [], 98 | bttv_emotes: BTTV_EMOTES ? BTTV_EMOTES : [], 99 | "7tv_emotes": _7TV_EMOTES ? _7TV_EMOTES : [], 100 | }) 101 | .then(() => console.info(`Created ${vodId} emotes..`)) 102 | .catch((e) => console.error(e)); 103 | else 104 | await app 105 | .service("emotes") 106 | .patch(vodId, { 107 | ffz_emotes: FFZ_EMOTES ? FFZ_EMOTES : [], 108 | bttv_emotes: BTTV_EMOTES ? BTTV_EMOTES : [], 109 | "7tv_emotes": _7TV_EMOTES ? _7TV_EMOTES : [], 110 | }) 111 | .then(() => console.info(`Patched ${vodId} emotes..`)) 112 | .catch((e) => console.error(e)); 113 | }; 114 | -------------------------------------------------------------------------------- /src/middleware/ffmpeg.js: -------------------------------------------------------------------------------- 1 | const ffmpeg = require("fluent-ffmpeg"); 2 | const readline = require("readline"); 3 | 4 | module.exports.mp4Download = async (m3u8, path) => { 5 | return new Promise((resolve, reject) => { 6 | const ffmpeg_process = ffmpeg(m3u8); 7 | ffmpeg_process 8 | .videoCodec("copy") 9 | .audioCodec("copy") 10 | .outputOptions(["-bsf:a aac_adtstoasc"]) 11 | .toFormat("mp4") 12 | .on("progress", (progress) => { 13 | if ((process.env.NODE_ENV || "").trim() !== "production") { 14 | readline.clearLine(process.stdout, 0); 15 | readline.cursorTo(process.stdout, 0, null); 16 | process.stdout.write( 17 | `DOWNLOAD PROGRESS: ${Math.round(progress.percent)}%` 18 | ); 19 | } 20 | }) 21 | .on("start", (cmd) => { 22 | console.info(`Starting m3u8 download for ${m3u8} in ${path}`); 23 | }) 24 | .on("error", function (err) { 25 | ffmpeg_process.kill("SIGKILL"); 26 | reject(err); 27 | }) 28 | .on("end", function () { 29 | resolve(); 30 | }) 31 | .saveToFile(path); 32 | }); 33 | }; 34 | 35 | module.exports.getDuration = async (video) => { 36 | let duration; 37 | await new Promise((resolve, reject) => { 38 | ffmpeg.ffprobe(video, (err, metadata) => { 39 | if (err) { 40 | console.error(err); 41 | return reject(); 42 | } 43 | duration = metadata.format.duration; 44 | resolve(); 45 | }); 46 | }); 47 | return Math.round(duration); 48 | }; 49 | -------------------------------------------------------------------------------- /src/middleware/index.js: -------------------------------------------------------------------------------- 1 | const admin = require("./admin"); 2 | const logs = require("./logs"); 3 | const live = require("./live"); 4 | const youtube = require("./youtube"); 5 | const twitch = require("./twitch"); 6 | const { limiter } = require("./rateLimit"); 7 | const dmca = require("./dmca"); 8 | 9 | module.exports = function (app) { 10 | app.post("/admin/download", limiter(app), admin.verify(app), admin.download(app)); 11 | app.post("/admin/hls/download", limiter(app), admin.verify(app), admin.hlsDownload(app)); 12 | app.post("/admin/manual/vod", limiter(app), admin.verify(app), admin.vodUpload(app)); 13 | app.post("/admin/manual/game", limiter(app), admin.verify(app), admin.gameUpload(app)); 14 | app.post("/admin/logs", limiter(app), admin.verify(app), admin.logs(app)); 15 | app.post("/admin/logs/manual", limiter(app), admin.verify(app), admin.manualLogs(app)); 16 | app.post("/admin/dmca", limiter(app), admin.verify(app), dmca(app)); 17 | app.post("/admin/create", limiter(app), admin.verify(app), admin.createVod(app)); 18 | app.delete("/admin/delete", limiter(app), admin.verify(app), admin.deleteVod(app)); 19 | app.post("/admin/part/dmca", limiter(app), admin.verify(app), dmca.part(app)); 20 | app.post("/admin/chapters", limiter(app), admin.verify(app), admin.saveChapters(app)); 21 | app.post("/admin/duration", limiter(app), admin.verify(app), admin.saveDuration(app)); 22 | app.post("/admin/reupload", limiter(app), admin.verify(app), admin.reUploadPart(app)); 23 | app.post("/admin/reupload/game", limiter(app), admin.verify(app), admin.reuploadGame(app)); 24 | app.post("/admin/youtube/parts", limiter(app), admin.verify(app), youtube.parts(app)); 25 | app.post("/admin/youtube/chapters", limiter(app), admin.verify(app), youtube.chapters(app)); 26 | app.post("/admin/games", limiter(app), admin.verify(app), admin.addGame(app)); 27 | app.post("/admin/emotes", limiter(app), admin.verify(app), admin.saveEmotes(app)); 28 | app.post("/v2/live", limiter(app), admin.verify(app), live(app)); 29 | app.get("/v2/badges", limiter(app), twitch.badges(app)); 30 | app.get("/v1/vods/:vodId/comments", limiter(app), logs(app)); 31 | }; 32 | -------------------------------------------------------------------------------- /src/middleware/kick.js: -------------------------------------------------------------------------------- 1 | const axios = require("axios"); 2 | const HLS = require("hls-parser"); 3 | const config = require("../../config/config.json"); 4 | const ffmpeg = require("./ffmpeg"); 5 | const dayjs = require("dayjs"); 6 | const utc = require("dayjs/plugin/utc"); 7 | const duration = require("dayjs/plugin/duration"); 8 | dayjs.extend(duration); 9 | dayjs.extend(utc); 10 | const readline = require("readline"); 11 | const fs = require("fs"); 12 | const vodFunc = require("./vod"); 13 | const drive = require("./drive"); 14 | const youtube = require("./youtube"); 15 | 16 | module.exports.getChannel = async (app, username) => { 17 | const browser = app.get("puppeteer"); 18 | if (!browser) return; 19 | const page = await browser.newPage(); 20 | 21 | await page 22 | .goto(`https://kick.com/api/v2/channels/${username}`, { 23 | waitUntil: "domcontentloaded", 24 | }) 25 | .catch((err) => { 26 | console.error(err); 27 | return undefined; 28 | }); 29 | await sleep(10000); 30 | await page.content(); 31 | const jsonContent = await page.evaluate(() => { 32 | try { 33 | return JSON.parse(document.querySelector("body").innerText); 34 | } catch { 35 | console.error("Kick: Failed to parse json"); 36 | return undefined; 37 | } 38 | }); 39 | 40 | await page.close(); 41 | 42 | return jsonContent; 43 | }; 44 | 45 | module.exports.getStream = async (app, username) => { 46 | const browser = app.get("puppeteer"); 47 | if (!browser) return; 48 | const page = await browser.newPage(); 49 | 50 | await page 51 | .goto(`https://kick.com/api/v2/channels/${username}/livestream`, { 52 | waitUntil: "domcontentloaded", 53 | }) 54 | .catch((err) => { 55 | console.error(err); 56 | return undefined; 57 | }); 58 | await sleep(10000); 59 | await page.content(); 60 | const jsonContent = await page.evaluate(() => { 61 | try { 62 | return JSON.parse(document.querySelector("body").innerText); 63 | } catch { 64 | console.error("Kick: Failed to parse json"); 65 | return undefined; 66 | } 67 | }); 68 | 69 | await page.close(); 70 | 71 | return jsonContent; 72 | }; 73 | 74 | module.exports.getVods = async (app, username) => { 75 | const browser = app.get("puppeteer"); 76 | if (!browser) return; 77 | const page = await browser.newPage(); 78 | 79 | await page 80 | .goto(`https://kick.com/api/v2/channels/${username}/videos`, { 81 | waitUntil: "domcontentloaded", 82 | }) 83 | .catch((err) => { 84 | console.error(err); 85 | return undefined; 86 | }); 87 | await sleep(10000); 88 | await page.content(); 89 | const jsonContent = await page.evaluate(() => { 90 | try { 91 | return JSON.parse(document.querySelector("body").innerText); 92 | } catch { 93 | console.error("Kick: Failed to parse json"); 94 | return undefined; 95 | } 96 | }); 97 | 98 | await page.close(); 99 | 100 | return jsonContent; 101 | }; 102 | 103 | module.exports.getVod = async (app, username, vodId) => { 104 | const browser = app.get("puppeteer"); 105 | if (!browser) return; 106 | const page = await browser.newPage(); 107 | 108 | await page 109 | .goto(`https://kick.com/api/v2/channels/${username}/videos`, { 110 | waitUntil: "domcontentloaded", 111 | }) 112 | .catch((err) => { 113 | console.error(err); 114 | return undefined; 115 | }); 116 | await sleep(10000); 117 | await page.content(); 118 | const jsonContent = await page.evaluate(() => { 119 | try { 120 | return JSON.parse(document.querySelector("body").innerText); 121 | } catch { 122 | console.error("Kick: Failed to parse json"); 123 | return undefined; 124 | } 125 | }); 126 | 127 | if (!jsonContent) return null; 128 | 129 | const vod = jsonContent.find( 130 | (livestream) => livestream.id.toString() === vodId 131 | ); 132 | 133 | await page.close(); 134 | 135 | return vod; 136 | }; 137 | 138 | module.exports.downloadMP4 = async (app, username, vodId) => { 139 | const vod = await this.getVod(app, username, vodId); 140 | if (!vod) return null; 141 | 142 | let m3u8 = await this.getM3u8(vod.source); 143 | if (!m3u8) return null; 144 | 145 | const baseURL = vod.source.replace("/master.m3u8", ""); 146 | m3u8 = this.getParsedM3u8(m3u8, baseURL); 147 | if (!m3u8) return null; 148 | 149 | const vodPath = `${config.vodPath}/${vodId}.mp4`; 150 | 151 | const success = await ffmpeg 152 | .mp4Download(m3u8, vodPath) 153 | .then(() => { 154 | console.info(`Downloaded ${vodId}.mp4\n`); 155 | return true; 156 | }) 157 | .catch((e) => { 158 | console.error("\nffmpeg error occurred: " + e); 159 | return false; 160 | }); 161 | 162 | if (success) return vodPath; 163 | 164 | return null; 165 | }; 166 | 167 | module.exports.getM3u8 = async (source) => { 168 | const data = await axios 169 | .get(source) 170 | .then((response) => response.data) 171 | .catch((e) => { 172 | console.error(e.response ? e.response.data : e); 173 | return null; 174 | }); 175 | return data; 176 | }; 177 | 178 | module.exports.getParsedM3u8 = (m3u8, baseURL) => { 179 | let parsedM3u8; 180 | try { 181 | parsedM3u8 = HLS.parse(m3u8); 182 | } catch (e) { 183 | console.error(e); 184 | } 185 | return parsedM3u8 ? `${baseURL}/${parsedM3u8.variants[0].uri}` : null; 186 | }; 187 | 188 | const fetchComments = async (app, start_time) => { 189 | const browser = app.get("puppeteer"); 190 | if (!browser) return; 191 | const page = await browser.newPage(); 192 | 193 | await page 194 | .goto( 195 | `https://kick.com/api/v2/channels/${config.kick.id}/messages?start_time=${start_time}`, 196 | { waitUntil: "domcontentloaded" } 197 | ) 198 | .catch((err) => { 199 | console.error(err); 200 | return undefined; 201 | }); 202 | await sleep(10000); 203 | await page.content(); 204 | const jsonContent = await page.evaluate(() => { 205 | try { 206 | return JSON.parse(document.querySelector("body").innerText); 207 | } catch { 208 | console.error("Kick: Failed to parse json"); 209 | return undefined; 210 | } 211 | }); 212 | await page.close(); 213 | 214 | return jsonContent; 215 | }; 216 | 217 | module.exports.downloadLogs = async (vodId, app, vod_start_date, duration) => { 218 | console.info(`Saving kick logs for ${vodId}`); 219 | let start_time = new Date(); 220 | let comments = []; 221 | let howMany = 1; 222 | let cursor = vod_start_date; 223 | 224 | const browser = app.get("puppeteer"); 225 | if (!browser) return; 226 | const page = await browser.newPage(); 227 | 228 | do { 229 | let response = await fetchComments(page, cursor); 230 | if (!response.data) { 231 | console.info(`No comments for vod ${vodId}`); 232 | return; 233 | } 234 | 235 | let responseComments = response.data.messages; 236 | const lastComment = responseComments[responseComments.length - 1]; 237 | cursor = lastComment.created_at; 238 | let currentDuration = dayjs(lastComment.created_at).diff( 239 | dayjs.utc(vod_start_date), 240 | "second" 241 | ); 242 | if ((process.env.NODE_ENV || "").trim() !== "production") { 243 | readline.clearLine(process.stdout, 0); 244 | readline.cursorTo(process.stdout, 0, null); 245 | process.stdout.write( 246 | `Current Log position: ${dayjs 247 | .duration(currentDuration, "s") 248 | .format("HH:mm:ss")}` 249 | ); 250 | } 251 | if (currentDuration >= duration / 1000) break; 252 | 253 | for (let comment of responseComments) { 254 | if (await commentExists(comment.id, app)) continue; 255 | if (comments.length >= 2500) { 256 | await app 257 | .service("logs") 258 | .create(comments) 259 | .then(() => { 260 | if ((process.env.NODE_ENV || "").trim() !== "production") { 261 | console.info( 262 | `\nSaved ${comments.length} comments in DB for vod ${vodId}` 263 | ); 264 | } 265 | }) 266 | .catch((e) => { 267 | console.error(e); 268 | }); 269 | comments = []; 270 | } 271 | 272 | const commenter = comment.sender; 273 | comments.push({ 274 | id: comment.id, 275 | vod_id: vodId, 276 | display_name: commenter.username, 277 | content_offset_seconds: dayjs(comment.created_at).diff( 278 | dayjs(vod_start_date), 279 | "second" 280 | ), 281 | message: comment.content, 282 | user_badges: commenter.identity.badges, 283 | user_color: commenter.identity.color, 284 | createdAt: comment.created_at, 285 | }); 286 | } 287 | howMany++; 288 | await sleep(25); 289 | } while (true); 290 | 291 | await app 292 | .service("logs") 293 | .create(comments) 294 | .then(() => { 295 | console.info(`Saved all kick comments in DB for vod ${vodId}`); 296 | }) 297 | .catch(() => {}); 298 | 299 | console.info( 300 | `\nTotal API Calls: ${howMany} | Total Time to get logs for ${vodId}: ${ 301 | (new Date() - start_time) / 1000 302 | } seconds` 303 | ); 304 | }; 305 | 306 | const commentExists = async (id, app) => { 307 | const exists = await app 308 | .service("logs") 309 | .get(id) 310 | .then(() => true) 311 | .catch(() => false); 312 | return exists; 313 | }; 314 | 315 | const sleep = (ms) => { 316 | return new Promise((resolve) => setTimeout(resolve, ms)); 317 | }; 318 | 319 | const getChapterInfo = async (app, chapter) => { 320 | const browser = app.get("puppeteer"); 321 | if (!browser) return; 322 | const page = await browser.newPage(); 323 | 324 | await page 325 | .goto(`https://kick.com/api/v1/subcategories/${chapter}`, { 326 | waitUntil: "domcontentloaded", 327 | }) 328 | .catch((err) => { 329 | console.error(err); 330 | return undefined; 331 | }); 332 | await sleep(10000); 333 | await page.content(); 334 | const jsonContent = await page.evaluate(() => { 335 | try { 336 | return JSON.parse(document.querySelector("body").innerText); 337 | } catch { 338 | console.error("Kick: Failed to parse json"); 339 | return undefined; 340 | } 341 | }); 342 | 343 | await page.close(); 344 | 345 | return jsonContent; 346 | }; 347 | 348 | module.exports.saveChapters = async (stream, app) => { 349 | const chapters = await app 350 | .service("vods") 351 | .get(stream.id.toString()) 352 | .then((vod) => vod.chapters) 353 | .catch(() => null); 354 | 355 | if (!chapters) return; 356 | const currentChapter = stream.category; 357 | const lastChapter = chapters[chapters.length - 1]; 358 | const currentTime = dayjs.duration(dayjs.utc().diff(stream.created_at)); 359 | if (lastChapter && lastChapter.gameId === currentChapter.id) { 360 | //Same chapter still, only save end time. 361 | lastChapter.end = Math.round(currentTime.asSeconds() - lastChapter.start); 362 | } else { 363 | //New chapter 364 | const chapterInfo = await getChapterInfo(app, currentChapter.slug); 365 | chapters.push({ 366 | gameId: chapterInfo.id, 367 | name: chapterInfo.name, 368 | image: chapterInfo.banner.src, 369 | duration: 370 | chapters.length === 0 371 | ? "00:00:00" 372 | : dayjs 373 | .duration(currentTime.asSeconds() - lastChapter.start, "s") 374 | .format("HH:mm:ss"), 375 | start: chapters.length === 0 ? 0 : Math.round(currentTime.asSeconds()), 376 | }); 377 | 378 | //Update end to last chapter when new chapter is found. 379 | if (lastChapter) 380 | lastChapter.end = Math.round(currentTime.asSeconds() - lastChapter.start); 381 | } 382 | 383 | await app 384 | .service("vods") 385 | .patch(stream.id.toString(), { 386 | chapters: chapters, 387 | }) 388 | .catch((e) => { 389 | console.error(e); 390 | }); 391 | }; 392 | 393 | module.exports.downloadHLS = async ( 394 | vodId, 395 | app, 396 | source, 397 | retry = 0, 398 | delay = 1 399 | ) => { 400 | if ((process.env.NODE_ENV || "").trim() !== "production") 401 | console.info(`${vodId} Download Retry: ${retry}`); 402 | const dir = `${config.vodPath}/${vodId}`; 403 | const m3u8Path = `${dir}/${vodId}.m3u8`; 404 | const stream = await this.getStream(app, config.kick.username); 405 | const m3u8Exists = await fileExists(m3u8Path); 406 | let duration, vod; 407 | await app 408 | .service("vods") 409 | .get(vodId) 410 | .then((data) => { 411 | vod = data; 412 | }) 413 | .catch(() => {}); 414 | 415 | if (!vod) 416 | return console.error("Failed to download video: no VOD in database"); 417 | 418 | if (m3u8Exists) { 419 | duration = await ffmpeg.getDuration(m3u8Path); 420 | await saveDuration(vodId, duration, app); 421 | if (stream && stream.data) await this.saveChapters(stream.data, app); 422 | } 423 | 424 | if ( 425 | duration >= config.youtube.splitDuration && 426 | config.youtube.liveUpload && 427 | config.youtube.upload 428 | ) { 429 | const noOfParts = Math.floor(duration / config.youtube.splitDuration); 430 | 431 | const vod_youtube_data = vod.youtube.filter((data) => { 432 | return data.type === "vod"; 433 | }); 434 | if (vod_youtube_data.length < noOfParts) { 435 | for (let i = 0; i < noOfParts; i++) { 436 | if (vod_youtube_data[i]) continue; 437 | await vodFunc.liveUploadPart( 438 | app, 439 | vodId, 440 | m3u8Path, 441 | config.youtube.splitDuration * i, 442 | config.youtube.splitDuration, 443 | i + 1 444 | ); 445 | } 446 | } 447 | } 448 | 449 | if (retry >= 10) { 450 | app.set(`${config.channel}-${vodId}-vod-downloading`, false); 451 | 452 | const mp4Path = `${config.vodPath}/${vodId}.mp4`; 453 | await vodFunc.convertToMp4(m3u8Path, vodId, mp4Path); 454 | if (config.drive.upload) await drive.upload(vodId, mp4Path, app); 455 | if (config.youtube.liveUpload && config.youtube.upload) { 456 | //upload last part 457 | let startTime = 0; 458 | 459 | const vod_youtube_data = vod.youtube.filter( 460 | (data) => data.type === "vod" 461 | ); 462 | for (let i = 0; i < vod_youtube_data.length; i++) { 463 | startTime += vod_youtube_data[i].duration; 464 | } 465 | await vodFunc.liveUploadPart( 466 | app, 467 | vodId, 468 | m3u8Path, 469 | startTime, 470 | duration - startTime, 471 | vod_youtube_data.length + 1 472 | ); 473 | //save parts at last upload. 474 | setTimeout(() => youtube.saveParts(vodId, app, "vod"), 60000); 475 | } else if (config.youtube.upload) { 476 | await vodFunc.upload(vodId, app, mp4Path); 477 | if (!config.saveMP4) await fs.promises.rm(mp4Path); 478 | } 479 | if (!config.saveHLS) 480 | await fs.promises.rm(dir, { 481 | recursive: true, 482 | }); 483 | return; 484 | } 485 | 486 | //Make variant work with 1080p playlist 487 | let baseURL; 488 | if (source.includes("master.m3u8")) { 489 | baseURL = `${source.substring(0, source.lastIndexOf("/"))}/1080p60`; 490 | } else { 491 | baseURL = `${source.substring(0, source.lastIndexOf("/"))}`; 492 | } 493 | 494 | let m3u8 = await this.getM3u8(`${baseURL}/playlist.m3u8`); 495 | if (!m3u8) { 496 | setTimeout(() => { 497 | this.downloadHLS(vodId, app, source, retry, delay); 498 | }, 1000 * 60 * delay); 499 | return console.error(`failed to get m3u8 for ${vodId}`); 500 | } 501 | 502 | m3u8 = HLS.parse(m3u8); 503 | 504 | if (!(await fileExists(m3u8Path))) { 505 | if (!(await fileExists(dir))) { 506 | fs.mkdirSync(dir); 507 | } 508 | await downloadTSFiles(m3u8, dir, baseURL, vodId); 509 | 510 | setTimeout(() => { 511 | this.downloadHLS(vodId, app, source, retry, delay); 512 | }, 1000 * 60 * delay); 513 | return; 514 | } 515 | 516 | let videoM3u8 = await fs.promises.readFile(m3u8Path, "utf8").catch((e) => { 517 | console.error(e); 518 | return null; 519 | }); 520 | 521 | if (!videoM3u8) { 522 | setTimeout(() => { 523 | this.downloadHLS(vodId, app, source, retry, delay); 524 | }, 1000 * 60 * delay); 525 | return; 526 | } 527 | 528 | videoM3u8 = HLS.parse(videoM3u8); 529 | 530 | //retry if last segment is the same as on file m3u8 and if the actual segment exists. 531 | if ( 532 | m3u8.segments[m3u8.segments.length - 1].uri === 533 | videoM3u8.segments[videoM3u8.segments.length - 1].uri && 534 | (await fileExists(`${dir}/${m3u8.segments[m3u8.segments.length - 1].uri}`)) 535 | ) { 536 | retry++; 537 | setTimeout(() => { 538 | this.downloadHLS(vodId, app, source, retry, delay); 539 | }, 1000 * 60 * delay); 540 | return; 541 | } 542 | 543 | //reset retry if downloading new ts files. 544 | retry = 1; 545 | await downloadTSFiles(m3u8, dir, baseURL, vodId); 546 | 547 | setTimeout(() => { 548 | this.downloadHLS(vodId, app, source, retry, delay); 549 | }, 1000 * 60 * delay); 550 | }; 551 | 552 | const saveDuration = async (vodId, duration, app) => { 553 | duration = toHHMMSS(duration); 554 | 555 | await app 556 | .service("vods") 557 | .patch(vodId, { 558 | duration: duration, 559 | }) 560 | .catch((e) => { 561 | console.error(e); 562 | }); 563 | }; 564 | 565 | const downloadTSFiles = async (m3u8, dir, baseURL, vodId) => { 566 | try { 567 | fs.writeFileSync(`${dir}/${vodId}.m3u8`, HLS.stringify(m3u8)); 568 | } catch (err) { 569 | console.error(err); 570 | } 571 | for (let segment of m3u8.segments) { 572 | if (await fileExists(`${dir}/${segment.uri}`)) continue; 573 | 574 | await axios({ 575 | method: "get", 576 | url: `${baseURL}/${segment.uri}`, 577 | responseType: "stream", 578 | }) 579 | .then((response) => { 580 | if ((process.env.NODE_ENV || "").trim() !== "production") { 581 | console.info(`Downloaded ${segment.uri}`); 582 | } 583 | response.data.pipe(fs.createWriteStream(`${dir}/${segment.uri}`)); 584 | }) 585 | .catch((e) => { 586 | console.error(e); 587 | }); 588 | } 589 | if ((process.env.NODE_ENV || "").trim() !== "production") { 590 | console.info( 591 | `Done downloading.. Last segment was ${ 592 | m3u8.segments[m3u8.segments.length - 1].uri 593 | }` 594 | ); 595 | } 596 | }; 597 | 598 | const fileExists = async (file) => { 599 | return fs.promises 600 | .access(file, fs.constants.F_OK) 601 | .then(() => true) 602 | .catch(() => false); 603 | }; 604 | 605 | const toHHMMSS = (secs) => { 606 | var sec_num = parseInt(secs, 10); 607 | var hours = Math.floor(sec_num / 3600); 608 | var minutes = Math.floor(sec_num / 60) % 60; 609 | var seconds = sec_num % 60; 610 | 611 | return [hours, minutes, seconds] 612 | .map((v) => (v < 10 ? "0" + v : v)) 613 | .filter((v, i) => v !== "00" || i > 0) 614 | .join(":"); 615 | }; 616 | -------------------------------------------------------------------------------- /src/middleware/live.js: -------------------------------------------------------------------------------- 1 | const vod = require("./vod"); 2 | const drive = require("./drive"); 3 | const config = require("../../config/config.json"); 4 | 5 | module.exports = function (app) { 6 | return async function (req, res, next) { 7 | const { streamId, path, driveId, platform } = req.body; 8 | if (!streamId) 9 | return res.status(400).json({ error: true, msg: "No streamId" }); 10 | 11 | if (!path) return res.status(400).json({ error: true, msg: "No Path" }); 12 | 13 | let vods; 14 | await app 15 | .service("vods") 16 | .find({ 17 | query: { 18 | stream_id: streamId, 19 | }, 20 | }) 21 | .then((data) => { 22 | vods = data.data; 23 | }) 24 | .catch((e) => { 25 | console.error(e); 26 | }); 27 | 28 | if (vods.length == 0) 29 | return res.status(404).json({ error: true, msg: "No Vod found" }); 30 | 31 | const vod_data = vods[0]; 32 | 33 | if (driveId == null && config.drive.upload) { 34 | drive.upload(vod_data.id, path, app, "live"); 35 | } else if (driveId != null) { 36 | vod_data.drive.push({ 37 | id: driveId, 38 | type: "live", 39 | }); 40 | await app 41 | .service("vods") 42 | .patch(vod_data.id, { 43 | drive: vod_data.drive, 44 | }) 45 | .then(() => { 46 | console.info(`Drive info updated for ${vod_data.id}`); 47 | }) 48 | .catch((e) => { 49 | console.error(e); 50 | }); 51 | } 52 | 53 | //Need to deliver a non 200 http code so it will delete the file 54 | if (config.youtube.multiTrack) { 55 | res.status(200).json({ error: false, msg: "Starting upload to youtube" }); 56 | if (platform) { 57 | await vod.upload(vod_data.id, app, path, "live", platform); 58 | } else { 59 | await vod.upload(vod_data.id, app, path, "live"); 60 | } 61 | } else { 62 | res.status(404).json({ 63 | error: true, 64 | msg: "Not Uploading to youtube as per multitrack var", 65 | }); 66 | } 67 | }; 68 | }; 69 | -------------------------------------------------------------------------------- /src/middleware/logs.js: -------------------------------------------------------------------------------- 1 | const config = require("../../config/config.json"); 2 | 3 | module.exports = function (app) { 4 | return async function (req, res, next) { 5 | if (!req.params.vodId) 6 | return res 7 | .status(400) 8 | .json({ error: true, msg: "Missing request params" }); 9 | if (!req.query.content_offset_seconds && !req.query.cursor) 10 | return res 11 | .status(400) 12 | .json({ error: true, msg: "Missing request params" }); 13 | 14 | const vodId = req.params.vodId, 15 | content_offset_seconds = parseFloat( 16 | req.query.content_offset_seconds 17 | ).toFixed(1), 18 | cursor = req.query.cursor; 19 | 20 | const client = app.get("redisClient"); 21 | let responseJson; 22 | 23 | if (!isNaN(content_offset_seconds) && content_offset_seconds !== null) { 24 | const vodData = await returnVodData(app, vodId); 25 | if (!vodData) 26 | return res.status(500).json({ 27 | error: true, 28 | msg: `Failed to retrieve vod ${vodId}`, 29 | }); 30 | 31 | let key = `${config.channel}-${vodId}-offset-${content_offset_seconds}`; 32 | responseJson = await client 33 | .get(key) 34 | .then((data) => JSON.parse(data)) 35 | .catch(() => null); 36 | 37 | if (!responseJson) { 38 | responseJson = await offsetSearch( 39 | app, 40 | vodId, 41 | content_offset_seconds, 42 | vodData 43 | ); 44 | 45 | if (!responseJson) 46 | return res.status(500).json({ 47 | error: true, 48 | msg: `Failed to retrieve comments from offset ${content_offset_seconds}`, 49 | }); 50 | 51 | client.set(key, JSON.stringify(responseJson), { 52 | EX: 60 * 5, 53 | }); 54 | } 55 | } else { 56 | let key = cursor; 57 | responseJson = await client 58 | .get(key) 59 | .then((data) => JSON.parse(data)) 60 | .catch(() => null); 61 | 62 | if (!responseJson) { 63 | let cursorJson; 64 | 65 | try { 66 | cursorJson = JSON.parse(Buffer.from(key, "base64").toString()); 67 | } catch (e) {} 68 | 69 | if (!cursorJson) 70 | return res 71 | .status(500) 72 | .json({ error: true, msg: "Failed to parse cursor" }); 73 | 74 | responseJson = await cursorSearch(app, vodId, cursorJson); 75 | 76 | if (!responseJson) 77 | return res.status(500).json({ 78 | error: true, 79 | msg: `Failed to retrieve comments from cursor ${cursor}`, 80 | }); 81 | 82 | client.set(key, JSON.stringify(responseJson), { 83 | EX: 60 * 60 * 24 * 1, 84 | }); 85 | } 86 | } 87 | 88 | return res.json(responseJson); 89 | }; 90 | }; 91 | 92 | const cursorSearch = async (app, vodId, cursorJson) => { 93 | const data = await app 94 | .service("logs") 95 | .find({ 96 | paginate: false, 97 | query: { 98 | vod_id: vodId, 99 | _id: { 100 | $gte: cursorJson.id, 101 | }, 102 | createdAt: { 103 | $gte: cursorJson.createdAt, 104 | }, 105 | $limit: 201, 106 | $sort: { 107 | content_offset_seconds: 1, 108 | _id: 1, 109 | }, 110 | }, 111 | }) 112 | .catch((e) => { 113 | console.error(e); 114 | return null; 115 | }); 116 | 117 | if (!data) return null; 118 | 119 | if (data.length === 0) return null; 120 | 121 | let cursor, comments; 122 | 123 | if (data.length === 201) { 124 | cursor = Buffer.from( 125 | JSON.stringify({ 126 | id: data[200]._id, 127 | content_offset_seconds: data[200].content_offset_seconds, 128 | createdAt: cursorJson.createdAt, 129 | }) 130 | ).toString("base64"); 131 | } 132 | 133 | comments = data.slice(0, 200); 134 | 135 | return { comments: comments, cursor: cursor }; 136 | }; 137 | 138 | const offsetSearch = async (app, vodId, content_offset_seconds, vodData) => { 139 | const startingId = await returnStartingId(app, vodId, vodData); 140 | if (!startingId) return null; 141 | 142 | const commentId = await returnCommentId( 143 | app, 144 | vodId, 145 | content_offset_seconds, 146 | vodData 147 | ); 148 | if (!commentId) return null; 149 | 150 | let index = parseInt(commentId) - parseInt(startingId); 151 | index = Math.floor(index / 200) * 200; 152 | 153 | const searchCursor = parseInt(startingId) + index; 154 | 155 | const data = await app 156 | .service("logs") 157 | .find({ 158 | paginate: false, 159 | query: { 160 | vod_id: vodId, 161 | _id: { 162 | $gte: searchCursor, 163 | }, 164 | $limit: 201, 165 | $sort: { 166 | content_offset_seconds: 1, 167 | _id: 1, 168 | }, 169 | }, 170 | }) 171 | .catch((e) => { 172 | console.error(e); 173 | return null; 174 | }); 175 | 176 | if (!data) return null; 177 | 178 | if (data.length === 0) return null; 179 | 180 | let cursor, comments; 181 | 182 | if (data.length === 201) { 183 | cursor = Buffer.from( 184 | JSON.stringify({ 185 | id: data[200]._id, 186 | content_offset_seconds: data[200].content_offset_seconds, 187 | createdAt: vodData.createdAt, 188 | }) 189 | ).toString("base64"); 190 | } 191 | 192 | comments = data.slice(0, 200); 193 | 194 | return { comments: comments, cursor: cursor }; 195 | }; 196 | 197 | const returnCommentId = async (app, vodId, content_offset_seconds, vodData) => { 198 | let data = await app 199 | .service("logs") 200 | .find({ 201 | paginate: false, 202 | query: { 203 | vod_id: vodId, 204 | content_offset_seconds: { 205 | $gte: content_offset_seconds, 206 | }, 207 | createdAt: { 208 | $gte: vodData.createdAt, 209 | }, 210 | $limit: 1, 211 | $sort: { 212 | content_offset_seconds: 1, 213 | _id: 1, 214 | }, 215 | }, 216 | }) 217 | .catch((e) => { 218 | console.error(e); 219 | return null; 220 | }); 221 | 222 | if (!data) return null; 223 | 224 | if (data.length === 0) return null; 225 | 226 | return data[0]._id; 227 | }; 228 | 229 | const returnStartingId = async (app, vodId, vodData) => { 230 | const key = `${config.channel}-${vodId}-chat-startingId`; 231 | const client = app.get("redisClient"); 232 | let startingId = await client 233 | .get(key) 234 | .then((data) => data) 235 | .catch(() => null); 236 | 237 | if (!startingId) { 238 | let data = await app 239 | .service("logs") 240 | .find({ 241 | paginate: false, 242 | query: { 243 | vod_id: vodId, 244 | $limit: 1, 245 | createdAt: { 246 | $gte: vodData.createdAt, 247 | }, 248 | $sort: { 249 | content_offset_seconds: 1, 250 | _id: 1, 251 | }, 252 | }, 253 | }) 254 | .catch(() => null); 255 | 256 | if (!data) return null; 257 | 258 | if (data.length === 0) return null; 259 | 260 | startingId = data[0]._id; 261 | 262 | client.set(key, startingId, { 263 | EX: 60 * 60 * 24 * 1, 264 | }); 265 | } 266 | 267 | return startingId; 268 | }; 269 | 270 | const returnVodData = async (app, vodId) => { 271 | let data = await app 272 | .service("vods") 273 | .get(vodId) 274 | .catch((e) => { 275 | console.error(e); 276 | return null; 277 | }); 278 | 279 | return data; 280 | }; 281 | -------------------------------------------------------------------------------- /src/middleware/rateLimit.js: -------------------------------------------------------------------------------- 1 | module.exports.limiter = (app) => { 2 | return async function (req, res, next) { 3 | app 4 | .get("rateLimiter") 5 | .consume(req.get("cf-connecting-ip") || req.get("X-Real-IP") || req.ip) 6 | .then((rateLimiteRes) => { 7 | const headers = { 8 | "Retry-After": rateLimiteRes.msBeforeNext, 9 | "X-RateLimit-Limit": app.get("rateLimiter")._points, 10 | "X-RateLimit-Remaining": rateLimiteRes.remainingPoints, 11 | "X-RateLimit-Reset": Date.now() + rateLimiteRes.msBeforeNext, 12 | }; 13 | res.set(headers); 14 | next(); 15 | }) 16 | .catch(() => { 17 | res.status(429).json({ error: true, msg: "Too Many Requests" }); 18 | }); 19 | }; 20 | }; 21 | -------------------------------------------------------------------------------- /src/middleware/twitch.js: -------------------------------------------------------------------------------- 1 | const axios = require("axios"); 2 | const config = require("../../config/config.json"); 3 | const fs = require("fs"); 4 | const path = require("path"); 5 | const HLS = require("hls-parser"); 6 | 7 | module.exports.checkToken = async () => { 8 | await axios(`https://id.twitch.tv/oauth2/validate`, { 9 | method: "GET", 10 | headers: { 11 | Authorization: `Bearer ${config.twitch.auth.access_token}`, 12 | }, 13 | }) 14 | .then(() => true) 15 | .catch(async (e) => { 16 | if (e.response && e.response.status === 401) { 17 | console.info("Twitch App Token Expired"); 18 | await this.refreshToken(); 19 | } 20 | console.error(e.response ? e.response.data : e); 21 | }); 22 | }; 23 | 24 | module.exports.refreshToken = async () => { 25 | await axios 26 | .post( 27 | `https://id.twitch.tv/oauth2/token?client_id=${config.twitch.auth.client_id}&client_secret=${config.twitch.auth.client_secret}&grant_type=client_credentials` 28 | ) 29 | .then((response) => { 30 | const data = response.data; 31 | config.twitch.auth.access_token = data.access_token; 32 | fs.writeFile( 33 | path.resolve(__dirname, "../../config/config.json"), 34 | JSON.stringify(config, null, 4), 35 | (err) => { 36 | if (err) return console.error(err); 37 | console.info("Refreshed Twitch App Token"); 38 | } 39 | ); 40 | }) 41 | .catch((e) => { 42 | console.error(e.response ? e.response.data : e); 43 | }); 44 | }; 45 | 46 | module.exports.getVodTokenSig = async (vodID) => { 47 | const data = await axios({ 48 | url: "https://gql.twitch.tv/gql", 49 | method: "POST", 50 | headers: { 51 | Accept: "*/*", 52 | "Client-Id": "kimne78kx3ncx6brgo4mv6wki5h1ko", //twitch's 53 | "Content-Type": "text/plain;charset=UTF-8", 54 | }, 55 | data: { 56 | operationName: "PlaybackAccessToken", 57 | variables: { 58 | isLive: false, 59 | login: "", 60 | isVod: true, 61 | vodID: vodID, 62 | platform: "web", 63 | playerBackend: "mediaplayer", 64 | playerType: "site", 65 | }, 66 | extensions: { 67 | persistedQuery: { 68 | version: 1, 69 | sha256Hash: 70 | "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712", 71 | }, 72 | }, 73 | }, 74 | }) 75 | .then((response) => response.data.data.videoPlaybackAccessToken) 76 | .catch((e) => { 77 | console.error(e.response ? e.response.data : e); 78 | return null; 79 | }); 80 | return data; 81 | }; 82 | 83 | module.exports.getM3u8 = async (vodId, token, sig) => { 84 | const data = await axios 85 | .get( 86 | `https://usher.ttvnw.net/vod/${vodId}.m3u8?allow_source=true&player=mediaplayer&include_unavailable=true&supported_codecs=av1,h265,h264&playlist_include_framerate=true&allow_spectre=true&nauthsig=${sig}&nauth=${token}` 87 | ) 88 | .then((response) => response.data) 89 | .catch((e) => { 90 | console.error(e.response ? e.response.data : e); 91 | return null; 92 | }); 93 | return data; 94 | }; 95 | 96 | module.exports.getParsedM3u8 = (m3u8) => { 97 | let parsedM3u8; 98 | try { 99 | parsedM3u8 = HLS.parse(m3u8); 100 | } catch (e) { 101 | console.error(e); 102 | } 103 | return parsedM3u8 ? parsedM3u8.variants[0].uri : null; 104 | }; 105 | 106 | module.exports.getVariantM3u8 = async (M3U8_URL) => { 107 | const data = await axios 108 | .get(M3U8_URL) 109 | .then((response) => response.data) 110 | .catch((e) => { 111 | console.error(e.response ? e.response.data : e); 112 | }); 113 | return data; 114 | }; 115 | 116 | module.exports.getLatestVodData = async (userId) => { 117 | await this.checkToken(); 118 | const vodData = await axios 119 | .get(`https://api.twitch.tv/helix/videos?user_id=${userId}`, { 120 | headers: { 121 | Authorization: `Bearer ${config.twitch.auth.access_token}`, 122 | "Client-Id": config.twitch.auth.client_id, 123 | }, 124 | }) 125 | .then((response) => response.data.data[0]) 126 | .catch((e) => { 127 | console.error(e.response ? e.response.data : e); 128 | return null; 129 | }); 130 | return vodData; 131 | }; 132 | 133 | module.exports.getVodData = async (vod_id) => { 134 | await this.checkToken(); 135 | const vodData = await axios 136 | .get(`https://api.twitch.tv/helix/videos?id=${vod_id}`, { 137 | headers: { 138 | Authorization: `Bearer ${config.twitch.auth.access_token}`, 139 | "Client-Id": config.twitch.auth.client_id, 140 | }, 141 | }) 142 | .then((response) => response.data.data[0]) 143 | .catch((e) => { 144 | console.error(e.response ? e.response.data : e); 145 | return null; 146 | }); 147 | return vodData; 148 | }; 149 | 150 | module.exports.getGameData = async (gameId) => { 151 | await this.checkToken(); 152 | const gameData = await axios 153 | .get(`https://api.twitch.tv/helix/games?id=${gameId}`, { 154 | headers: { 155 | Authorization: `Bearer ${config.twitch.auth.access_token}`, 156 | "Client-Id": config.twitch.auth.client_id, 157 | }, 158 | }) 159 | .then((response) => response.data.data[0]) 160 | .catch((e) => { 161 | console.error(e.response ? e.response.data : e); 162 | return null; 163 | }); 164 | return gameData; 165 | }; 166 | 167 | module.exports.fetchComments = async (vodId, offset = 0) => { 168 | const data = await axios({ 169 | url: "https://gql.twitch.tv/gql", 170 | method: "POST", 171 | headers: { 172 | Accept: "*/*", 173 | "Client-Id": "kimne78kx3ncx6brgo4mv6wki5h1ko", //twitch's 174 | "Content-Type": "text/plain;charset=UTF-8", 175 | }, 176 | data: { 177 | operationName: "VideoCommentsByOffsetOrCursor", 178 | variables: { 179 | videoID: vodId, 180 | contentOffsetSeconds: offset, 181 | }, 182 | extensions: { 183 | persistedQuery: { 184 | version: 1, 185 | sha256Hash: 186 | "b70a3591ff0f4e0313d126c6a1502d79a1c02baebb288227c582044aa76adf6a", 187 | }, 188 | }, 189 | }, 190 | }) 191 | .then((response) => response.data.data.video) 192 | .then((video) => { 193 | if (!video) return null; 194 | return video; 195 | }) 196 | .catch((e) => { 197 | console.error(e.response ? e.response.data : e); 198 | return null; 199 | }); 200 | return data; 201 | }; 202 | 203 | module.exports.fetchNextComments = async (vodId, cursor) => { 204 | const data = await axios({ 205 | url: "https://gql.twitch.tv/gql", 206 | method: "POST", 207 | headers: { 208 | Accept: "*/*", 209 | "Client-Id": "kd1unb4b3q4t58fwlpcbzcbnm76a8fp", 210 | "Content-Type": "text/plain;charset=UTF-8", 211 | }, 212 | data: { 213 | operationName: "VideoCommentsByOffsetOrCursor", 214 | variables: { 215 | videoID: vodId, 216 | cursor: cursor, 217 | }, 218 | extensions: { 219 | persistedQuery: { 220 | version: 1, 221 | sha256Hash: 222 | "b70a3591ff0f4e0313d126c6a1502d79a1c02baebb288227c582044aa76adf6a", 223 | }, 224 | }, 225 | }, 226 | }) 227 | .then((response) => response.data.data.video) 228 | .then((video) => { 229 | if (!video) return null; 230 | return video; 231 | }) 232 | .catch((e) => { 233 | console.error(e.response ? e.response.data : e); 234 | return null; 235 | }); 236 | return data; 237 | }; 238 | 239 | module.exports.getChapters = async (vodID) => { 240 | const data = await axios({ 241 | url: "https://gql.twitch.tv/gql", 242 | method: "POST", 243 | headers: { 244 | Accept: "*/*", 245 | "Client-Id": "kd1unb4b3q4t58fwlpcbzcbnm76a8fp", 246 | "Content-Type": "text/plain;charset=UTF-8", 247 | }, 248 | data: { 249 | operationName: "VideoPreviewCard__VideoMoments", 250 | variables: { 251 | videoId: vodID, 252 | }, 253 | extensions: { 254 | persistedQuery: { 255 | version: 1, 256 | sha256Hash: 257 | "0094e99aab3438c7a220c0b1897d144be01954f8b4765b884d330d0c0893dbde", 258 | }, 259 | }, 260 | }, 261 | }) 262 | .then((response) => { 263 | if (!response.data.data.video) return null; 264 | if (!response.data.data.video.moments) return null; 265 | return response.data.data.video.moments.edges; 266 | }) 267 | .catch((e) => { 268 | console.error(e.response ? e.response.data : e); 269 | return null; 270 | }); 271 | return data; 272 | }; 273 | 274 | module.exports.getChapter = async (vodID) => { 275 | const data = await axios({ 276 | url: "https://gql.twitch.tv/gql", 277 | method: "POST", 278 | headers: { 279 | Accept: "*/*", 280 | "Client-Id": "kimne78kx3ncx6brgo4mv6wki5h1ko", //twitch's 281 | "Content-Type": "text/plain;charset=UTF-8", 282 | }, 283 | data: { 284 | operationName: "NielsenContentMetadata", 285 | variables: { 286 | isCollectionContent: false, 287 | isLiveContent: false, 288 | isVODContent: true, 289 | collectionID: "", 290 | login: "", 291 | vodID: vodID, 292 | }, 293 | extensions: { 294 | persistedQuery: { 295 | version: 1, 296 | sha256Hash: 297 | "2dbf505ee929438369e68e72319d1106bb3c142e295332fac157c90638968586", 298 | }, 299 | }, 300 | }, 301 | }) 302 | .then((response) => response.data.data.video) 303 | .catch((e) => { 304 | console.error(e.response ? e.response.data : e); 305 | return null; 306 | }); 307 | return data; 308 | }; 309 | 310 | module.exports.getStream = async (twitchId) => { 311 | await this.checkToken(); 312 | const stream = await axios 313 | .get(`https://api.twitch.tv/helix/streams?user_id=${twitchId}`, { 314 | headers: { 315 | Authorization: `Bearer ${config.twitch.auth.access_token}`, 316 | "Client-Id": config.twitch.auth.client_id, 317 | }, 318 | }) 319 | .then((response) => response.data.data) 320 | .catch((e) => { 321 | console.error(e.response ? e.response.data : e); 322 | return null; 323 | }); 324 | return stream; 325 | }; 326 | 327 | module.exports.getChannelBadges = async () => { 328 | await this.checkToken(); 329 | const badges = await axios 330 | .get( 331 | `https://api.twitch.tv/helix/chat/badges?broadcaster_id=${config.twitch.id}`, 332 | { 333 | headers: { 334 | Authorization: `Bearer ${config.twitch.auth.access_token}`, 335 | "Client-Id": config.twitch.auth.client_id, 336 | }, 337 | } 338 | ) 339 | .then((response) => response.data.data) 340 | .catch((e) => { 341 | console.error(e.response ? e.response.data : e); 342 | return null; 343 | }); 344 | return badges; 345 | }; 346 | 347 | module.exports.getGlobalBadges = async () => { 348 | await this.checkToken(); 349 | const badges = await axios 350 | .get(`https://api.twitch.tv/helix/chat/badges/global`, { 351 | headers: { 352 | Authorization: `Bearer ${config.twitch.auth.access_token}`, 353 | "Client-Id": config.twitch.auth.client_id, 354 | }, 355 | }) 356 | .then((response) => response.data.data) 357 | .catch((e) => { 358 | console.error(e.response ? e.response.data : e); 359 | return null; 360 | }); 361 | return badges; 362 | }; 363 | 364 | module.exports.badges = function (app) { 365 | const _this = this; 366 | return async function (req, res, next) { 367 | const redisClient = app.get("redisClient"); 368 | const key = `${config.channel}-badges`; 369 | const cachedBadges = await redisClient 370 | .get(key) 371 | .then((data) => JSON.parse(data)) 372 | .catch(() => null); 373 | 374 | if (cachedBadges) return res.json(cachedBadges); 375 | 376 | let badges = { 377 | channel: await _this.getChannelBadges(), 378 | global: await _this.getGlobalBadges(), 379 | }; 380 | 381 | if (!badges) 382 | return res.status(500).json({ 383 | error: true, 384 | msg: "Something went wrong trying to retrieve channel badges..", 385 | }); 386 | 387 | res.json(badges); 388 | 389 | redisClient.set(key, JSON.stringify(badges), { 390 | EX: 3600, 391 | }); 392 | }; 393 | }; 394 | -------------------------------------------------------------------------------- /src/middleware/vod.js: -------------------------------------------------------------------------------- 1 | const ffmpeg = require("fluent-ffmpeg"); 2 | const twitch = require("./twitch"); 3 | const kick = require("./kick"); 4 | const config = require("../../config/config.json"); 5 | const fs = require("fs"); 6 | const readline = require("readline"); 7 | const path = require("path"); 8 | const HLS = require("hls-parser"); 9 | const axios = require("axios"); 10 | const drive = require("./drive"); 11 | const youtube = require("./youtube"); 12 | const emotes = require("./emotes"); 13 | const dayjs = require("dayjs"); 14 | const utc = require("dayjs/plugin/utc"); 15 | const timezone = require("dayjs/plugin/timezone"); 16 | const duration = require("dayjs/plugin/duration"); 17 | dayjs.extend(duration); 18 | dayjs.extend(utc); 19 | dayjs.extend(timezone); 20 | 21 | module.exports.upload = async ( 22 | vodId, 23 | app, 24 | manualPath = false, 25 | type = "vod" 26 | ) => { 27 | let vod; 28 | await app 29 | .service("vods") 30 | .get(vodId) 31 | .then((data) => { 32 | vod = data; 33 | }) 34 | .catch(() => {}); 35 | 36 | if (!vod) { 37 | console.error("Failed to download video: no VOD in database"); 38 | return; 39 | } 40 | 41 | let vodPath; 42 | 43 | if (manualPath) { 44 | vodPath = manualPath; 45 | } else if (type === "vod") { 46 | if (vod.platform === "twitch") { 47 | vodPath = await this.mp4Download(vodId); 48 | } else if (vod.platform === "kick") { 49 | vodPath = await kick.downloadMP4(app, config.kick.username, vodId); 50 | } 51 | } 52 | 53 | if (!vodPath && config.drive.enabled) { 54 | vodPath = await drive.download(vodId, type, app); 55 | } 56 | 57 | if (!vodPath) { 58 | console.error(`Could not find a download source for ${vodId}`); 59 | return; 60 | } 61 | 62 | if (config.youtube.perGameUpload && vod.chapters) { 63 | for (let chapter of vod.chapters) { 64 | if (chapter.end < 60 * 5) continue; 65 | if (config.youtube.restrictedGames.includes(chapter.name)) continue; 66 | 67 | console.info( 68 | `Trimming ${chapter.name} from ${vod.id} ${dayjs(vod.createdAt).format( 69 | "MM/DD/YYYY" 70 | )}` 71 | ); 72 | const trimmedPath = await this.trim( 73 | vodPath, 74 | vodId, 75 | chapter.start, 76 | chapter.end 77 | ); 78 | 79 | if (!trimmedPath) { 80 | console.error("Trim failed"); 81 | return; 82 | } 83 | 84 | if (chapter.end > config.youtube.splitDuration) { 85 | const duration = await getDuration(trimmedPath); 86 | let paths = await this.splitVideo(trimmedPath, duration, vodId); 87 | if (!paths) { 88 | console.error( 89 | "Something went wrong trying to split the trimmed video" 90 | ); 91 | return; 92 | } 93 | 94 | for (let i = 0; i < paths.length; i++) { 95 | let totalGames, gameTitle, ytTitle; 96 | 97 | await app 98 | .service("games") 99 | .find({ 100 | query: { 101 | game_name: chapter.name, 102 | $limit: 0, 103 | }, 104 | }) 105 | .then((response) => { 106 | totalGames = response.total; 107 | }) 108 | .catch((e) => { 109 | console.error(e); 110 | }); 111 | 112 | if (totalGames !== undefined) { 113 | ytTitle = `${config.channel} plays ${chapter.name} EP ${ 114 | totalGames + 1 115 | } - ${dayjs(vod.createdAt) 116 | .tz(config.timezone) 117 | .format("MMMM DD YYYY") 118 | .toUpperCase()}`; 119 | gameTitle = `${chapter.name} EP ${totalGames + 1}`; 120 | } else { 121 | ytTitle = `${config.channel} plays ${chapter.name} - ${dayjs( 122 | vod.createdAt 123 | ) 124 | .tz(config.timezone) 125 | .format("MMMM DD YYYY") 126 | .toUpperCase()} PART ${i + 1}`; 127 | gameTitle = `${chapter.name} PART ${i + 1}`; 128 | } 129 | 130 | await youtube.upload( 131 | { 132 | path: paths[i], 133 | title: ytTitle, 134 | gameTitle: gameTitle, 135 | type: "vod", 136 | public: true, 137 | duration: await getDuration(paths[i]), 138 | chapter: chapter, 139 | start_time: chapter.start + config.youtube.splitDuration * i, 140 | end_time: 141 | chapter.start + config.youtube.splitDuration * (i + 1) > 142 | chapter.end 143 | ? chapter.end 144 | : chapter.start + config.youtube.splitDuration * (i + 1), 145 | vod: vod, 146 | }, 147 | app, 148 | false 149 | ); 150 | fs.unlinkSync(paths[i]); 151 | } 152 | } else { 153 | let totalGames; 154 | await app 155 | .service("games") 156 | .find({ 157 | query: { 158 | game_name: chapter.name, 159 | $limit: 0, 160 | }, 161 | }) 162 | .then((response) => { 163 | totalGames = response.total; 164 | }) 165 | .catch((e) => { 166 | console.error(e); 167 | }); 168 | 169 | let gameTitle, ytTitle; 170 | if (totalGames !== undefined) { 171 | ytTitle = `${config.channel} plays ${chapter.name} EP ${ 172 | totalGames + 1 173 | } - ${dayjs(vod.createdAt) 174 | .tz(config.timezone) 175 | .format("MMMM DD YYYY") 176 | .toUpperCase()}`; 177 | gameTitle = `${chapter.name} EP ${totalGames + 1}`; 178 | } else { 179 | ytTitle = `${config.channel} plays ${chapter.name} - ${dayjs( 180 | vod.createdAt 181 | ) 182 | .tz(config.timezone) 183 | .format("MMMM DD YYYY") 184 | .toUpperCase()}`; 185 | gameTitle = `${chapter.name}`; 186 | } 187 | 188 | await youtube.upload( 189 | { 190 | path: trimmedPath, 191 | title: ytTitle, 192 | gameTitle: gameTitle, 193 | type: "vod", 194 | public: true, 195 | duration: await getDuration(trimmedPath), 196 | chapter: chapter, 197 | start_time: chapter.start, 198 | end_time: chapter.end, 199 | vod: vod, 200 | }, 201 | app, 202 | false 203 | ); 204 | fs.unlinkSync(trimmedPath); 205 | } 206 | } 207 | } 208 | 209 | if (config.youtube.vodUpload) { 210 | const duration = await getDuration(vodPath); 211 | await saveDuration(vodId, duration, app); 212 | await this.saveChapters(vodId, app, duration); 213 | 214 | if (duration > config.youtube.splitDuration) { 215 | let paths = await this.splitVideo(vodPath, duration, vodId); 216 | 217 | if (!paths) { 218 | console.error("Something went wrong trying to split the trimmed video"); 219 | return; 220 | } 221 | 222 | for (let i = 0; i < paths.length; i++) { 223 | const data = { 224 | path: paths[i], 225 | title: 226 | type === "vod" 227 | ? `${config.channel} ${ 228 | vod.platform.charAt(0).toUpperCase() + vod.platform.slice(1) 229 | } VOD - ${dayjs(vod.createdAt) 230 | .tz(config.timezone) 231 | .format("MMMM DD YYYY") 232 | .toUpperCase()} PART ${i + 1}` 233 | : `${config.channel} ${ 234 | vod.platform.charAt(0).toUpperCase() + vod.platform.slice(1) 235 | } Live VOD - ${dayjs(vod.createdAt) 236 | .tz(config.timezone) 237 | .format("MMMM DD YYYY") 238 | .toUpperCase()} PART ${i + 1}`, 239 | type: type, 240 | public: 241 | config.youtube.multiTrack && 242 | type === "live" && 243 | config.youtube.public 244 | ? true 245 | : !config.youtube.multiTrack && 246 | type === "vod" && 247 | config.youtube.public 248 | ? true 249 | : false, 250 | duration: await getDuration(paths[i]), 251 | vod: vod, 252 | part: i + 1, 253 | }; 254 | await youtube.upload(data, app); 255 | fs.unlinkSync(paths[i]); 256 | } 257 | setTimeout(async () => { 258 | await youtube.saveChapters(vodId, app, type); 259 | setTimeout(() => youtube.saveParts(vodId, app, type), 30000); 260 | }, 30000); 261 | if (config.drive.upload) fs.unlinkSync(vodPath); 262 | return vodPath; 263 | } 264 | 265 | const data = { 266 | path: vodPath, 267 | title: 268 | type === "vod" 269 | ? `${config.channel} ${ 270 | vod.platform.charAt(0).toUpperCase() + vod.platform.slice(1) 271 | } VOD - ${dayjs(vod.createdAt) 272 | .tz(config.timezone) 273 | .format("MMMM DD YYYY") 274 | .toUpperCase()}` 275 | : `${config.channel} ${ 276 | vod.platform.charAt(0).toUpperCase() + vod.platform.slice(1) 277 | } Live VOD - ${dayjs(vod.createdAt) 278 | .tz(config.timezone) 279 | .format("MMMM DD YYYY") 280 | .toUpperCase()}`, 281 | public: 282 | config.youtube.multiTrack && type === "live" && config.youtube.public 283 | ? true 284 | : !config.youtube.multiTrack && 285 | type === "vod" && 286 | config.youtube.public 287 | ? true 288 | : false, 289 | duration: duration, 290 | vod: vod, 291 | type: type, 292 | part: 1, 293 | }; 294 | 295 | await youtube.upload(data, app); 296 | setTimeout(async () => { 297 | await youtube.saveChapters(vodId, app, type); 298 | }, 30000); 299 | if (config.drive.upload) fs.unlinkSync(vodPath); 300 | return vodPath; 301 | } 302 | }; 303 | 304 | module.exports.manualVodUpload = async ( 305 | app, 306 | vodId, 307 | videoPath, 308 | type = "vod" 309 | ) => { 310 | let vod; 311 | await app 312 | .service("vods") 313 | .get(vodId) 314 | .then((data) => { 315 | vod = data; 316 | }) 317 | .catch(() => {}); 318 | 319 | if (!vod) return console.error("Failed to get vod: no VOD in database"); 320 | 321 | const duration = await getDuration(videoPath); 322 | 323 | const data = { 324 | path: videoPath, 325 | title: 326 | type === "vod" 327 | ? `${config.channel} ${ 328 | vod.platform.charAt(0).toUpperCase() + vod.platform.slice(1) 329 | } VOD - ${dayjs(vod.createdAt) 330 | .tz(config.timezone) 331 | .format("MMMM DD YYYY") 332 | .toUpperCase()}` 333 | : `${config.channel} ${ 334 | vod.platform.charAt(0).toUpperCase() + vod.platform.slice(1) 335 | } Live VOD - ${dayjs(vod.createdAt) 336 | .tz(config.timezone) 337 | .format("MMMM/DD/YYYY") 338 | .toUpperCase()}`, 339 | public: 340 | config.youtube.multiTrack && type === "live" && config.youtube.public 341 | ? true 342 | : !config.youtube.multiTrack && type === "vod" && config.youtube.public 343 | ? true 344 | : false, 345 | duration: duration, 346 | vod: vod, 347 | type: type, 348 | part: 1, 349 | }; 350 | 351 | await youtube.upload(data, app); 352 | setTimeout(async () => { 353 | await youtube.saveChapters(vodId, app, type); 354 | }, 30000); 355 | if (config.drive.upload) fs.unlinkSync(vodPath); 356 | }; 357 | 358 | module.exports.manualGameUpload = async (app, vod, game, videoPath) => { 359 | const { vodId, date, chapter } = game; 360 | const { name, end, start } = chapter; 361 | console.info( 362 | `Trimming ${name} from ${vodId} ${dayjs(date).format("MM/DD/YYYY")}` 363 | ); 364 | 365 | const trimmedPath = await this.trim(videoPath, vodId, start, end); 366 | if (!trimmedPath) return console.error("Trim failed"); 367 | 368 | if (end > config.youtube.splitDuration) { 369 | let paths = await this.splitVideo(trimmedPath, end, vodId); 370 | if (!paths) 371 | return console.error( 372 | "Something went wrong trying to split the trimmed video" 373 | ); 374 | 375 | for (let i = 0; i < paths.length; i++) { 376 | await youtube.upload( 377 | { 378 | path: paths[i], 379 | title: `${config.channel} plays ${name} - ${dayjs(date) 380 | .tz(config.timezone) 381 | .format("MMMM DD YYYY") 382 | .toUpperCase()} PART ${i + 1}`, 383 | type: "vod", 384 | public: true, 385 | duration: await getDuration(paths[i]), 386 | chapter: chapter, 387 | start_time: start + config.youtube.splitDuration * i, 388 | end_time: 389 | start + config.youtube.splitDuration * (i + 1) > end 390 | ? end 391 | : start + config.youtube.splitDuration * (i + 1), 392 | vod: vod, 393 | gameId: (game.gameId ? game.gameId + i : null ), 394 | }, 395 | app, 396 | false 397 | ); 398 | fs.unlinkSync(paths[i]); 399 | } 400 | } else { 401 | await youtube.upload( 402 | { 403 | path: trimmedPath, 404 | title: game.gameId ? game.title : `${config.channel} plays ${name} - ${dayjs(date) 405 | .tz(config.timezone) 406 | .format("MMMM DD YYYY") 407 | .toUpperCase()}`, 408 | type: "vod", 409 | public: true, 410 | duration: await getDuration(trimmedPath), 411 | chapter: chapter, 412 | start_time: start, 413 | end_time: end, 414 | vod: vod, 415 | gameId: game.gameId, 416 | }, 417 | app, 418 | false 419 | ); 420 | fs.unlinkSync(trimmedPath); 421 | } 422 | }; 423 | 424 | module.exports.liveUploadPart = async ( 425 | app, 426 | vodId, 427 | m3u8Path, 428 | start, 429 | end, 430 | part, 431 | type = "vod" 432 | ) => { 433 | let vod; 434 | await app 435 | .service("vods") 436 | .get(vodId) 437 | .then((data) => { 438 | vod = data; 439 | }) 440 | .catch(() => {}); 441 | 442 | if (!vod) 443 | return console.error("Failed in liveUploadPart: no VOD in database"); 444 | 445 | console.info( 446 | `Trimming ${vod.id} ${dayjs(vod.createdAt).format( 447 | "MM/DD/YYYY" 448 | )} | Start time: ${start} | Duration: ${end}` 449 | ); 450 | let trimmedPath = await this.trimHLS(m3u8Path, vodId, start, end); 451 | 452 | if (!trimmedPath) return console.error("Trim failed"); 453 | 454 | const data = { 455 | path: trimmedPath, 456 | title: 457 | type === "vod" 458 | ? `${config.channel} ${ 459 | vod.platform.charAt(0).toUpperCase() + vod.platform.slice(1) 460 | } VOD - ${dayjs(vod.createdAt) 461 | .tz(config.timezone) 462 | .format("MMMM DD YYYY") 463 | .toUpperCase()} PART ${part}` 464 | : `${config.channel} ${ 465 | vod.platform.charAt(0).toUpperCase() + vod.platform.slice(1) 466 | } Live VOD - ${dayjs(vod.createdAt) 467 | .tz(config.timezone) 468 | .format("MMMM DD YYYY") 469 | .toUpperCase()} PART ${part}`, 470 | public: 471 | config.youtube.multiTrack && type === "live" && config.youtube.public 472 | ? true 473 | : !config.youtube.multiTrack && type === "vod" && config.youtube.public 474 | ? true 475 | : false, 476 | duration: await getDuration(trimmedPath), 477 | vod: vod, 478 | type: type, 479 | part: part, 480 | }; 481 | 482 | await youtube.upload(data, app); 483 | setTimeout(async () => { 484 | await youtube.saveChapters(vodId, app, type); 485 | }, 30000); 486 | if (config.drive.upload) fs.unlinkSync(trimmedPath); 487 | }; 488 | 489 | module.exports.splitVideo = async (vodPath, duration, vodId) => { 490 | console.info(`Trying to split ${vodPath} with duration ${duration}`); 491 | const paths = []; 492 | for (let start = 0; start < duration; start += config.youtube.splitDuration) { 493 | await new Promise((resolve, reject) => { 494 | let cut = duration - start; 495 | if (cut > config.youtube.splitDuration) 496 | cut = config.youtube.splitDuration; 497 | const pathName = `${path.dirname(vodPath)}/${start}-${ 498 | cut + start 499 | }-${vodId}.mp4`; 500 | const ffmpeg_process = ffmpeg(vodPath); 501 | ffmpeg_process 502 | .videoCodec("copy") 503 | .audioCodec("copy") 504 | .outputOptions([`-ss ${start}`, "-copyts", `-t ${cut}`]) 505 | .toFormat("mp4") 506 | .on("progress", (progress) => { 507 | if ((process.env.NODE_ENV || "").trim() !== "production") { 508 | readline.clearLine(process.stdout, 0); 509 | readline.cursorTo(process.stdout, 0, null); 510 | process.stdout.write( 511 | `SPLIT VIDEO PROGRESS: ${Math.round(progress.percent)}%` 512 | ); 513 | } 514 | }) 515 | .on("start", (cmd) => { 516 | if ((process.env.NODE_ENV || "").trim() !== "production") { 517 | console.info(cmd); 518 | } 519 | console.info( 520 | `Splitting ${vodPath}. ${start} - ${ 521 | cut + start 522 | } with a duration of ${duration}` 523 | ); 524 | }) 525 | .on("error", function (err) { 526 | ffmpeg_process.kill("SIGKILL"); 527 | reject(err); 528 | }) 529 | .on("end", function () { 530 | resolve(pathName); 531 | }) 532 | .saveToFile(pathName); 533 | }) 534 | .then((argPath) => { 535 | paths.push(argPath); 536 | console.info("\n"); 537 | }) 538 | .catch((e) => { 539 | console.error("\nffmpeg error occurred: " + e); 540 | }); 541 | } 542 | return paths; 543 | }; 544 | 545 | module.exports.trim = async (vodPath, vodId, start, end) => { 546 | let returnPath; 547 | await new Promise((resolve, reject) => { 548 | const ffmpeg_process = ffmpeg(vodPath); 549 | ffmpeg_process 550 | .videoCodec("copy") 551 | .audioCodec("copy") 552 | .outputOptions([`-ss ${start}`, "-copyts", `-t ${end}`]) 553 | .toFormat("mp4") 554 | .on("progress", (progress) => { 555 | if ((process.env.NODE_ENV || "").trim() !== "production") { 556 | readline.clearLine(process.stdout, 0); 557 | readline.cursorTo(process.stdout, 0, null); 558 | process.stdout.write( 559 | `TRIM VIDEO PROGRESS: ${Math.round(progress.percent)}%` 560 | ); 561 | } 562 | }) 563 | .on("start", (cmd) => { 564 | if ((process.env.NODE_ENV || "").trim() !== "production") { 565 | console.info(cmd); 566 | } 567 | }) 568 | .on("error", function (err) { 569 | ffmpeg_process.kill("SIGKILL"); 570 | reject(err); 571 | }) 572 | .on("end", function () { 573 | resolve(`${path.dirname(vodPath)}/${vodId}-${start}-${end}.mp4`); 574 | }) 575 | .saveToFile(`${path.dirname(vodPath)}/${vodId}-${start}-${end}.mp4`); 576 | }) 577 | .then((result) => { 578 | returnPath = result; 579 | console.info("\n"); 580 | }) 581 | .catch((e) => { 582 | console.error("\nffmpeg error occurred: " + e); 583 | }); 584 | return returnPath; 585 | }; 586 | 587 | module.exports.trimHLS = async (vodPath, vodId, start, end) => { 588 | let returnPath; 589 | await new Promise((resolve, reject) => { 590 | const ffmpeg_process = ffmpeg(vodPath); 591 | ffmpeg_process 592 | .seekOutput(start) 593 | .videoCodec("copy") 594 | .audioCodec("copy") 595 | .outputOptions([ 596 | "-bsf:a aac_adtstoasc", 597 | "-copyts", 598 | "-start_at_zero", 599 | `-t ${end}`, 600 | ]) 601 | .toFormat("mp4") 602 | .on("progress", (progress) => { 603 | if ((process.env.NODE_ENV || "").trim() !== "production") { 604 | readline.clearLine(process.stdout, 0); 605 | readline.cursorTo(process.stdout, 0, null); 606 | process.stdout.write( 607 | `TRIM HLS VIDEO PROGRESS: ${Math.round(progress.percent)}%` 608 | ); 609 | } 610 | }) 611 | .on("start", (cmd) => { 612 | if ((process.env.NODE_ENV || "").trim() !== "production") { 613 | console.info(cmd); 614 | } 615 | }) 616 | .on("error", function (err) { 617 | ffmpeg_process.kill("SIGKILL"); 618 | reject(err); 619 | }) 620 | .on("end", function () { 621 | resolve(`${path.dirname(vodPath)}/${vodId}-${start}-${end}.mp4`); 622 | }) 623 | .saveToFile(`${path.dirname(vodPath)}/${vodId}-${start}-${end}.mp4`); 624 | }) 625 | .then((result) => { 626 | returnPath = result; 627 | console.info("\n"); 628 | }) 629 | .catch((e) => { 630 | console.error("\nffmpeg error occurred: " + e); 631 | }); 632 | return returnPath; 633 | }; 634 | 635 | const commentExists = async (id, app) => { 636 | const exists = await app 637 | .service("logs") 638 | .get(id) 639 | .then(() => true) 640 | .catch(() => false); 641 | return exists; 642 | }; 643 | 644 | const sleep = (ms) => { 645 | return new Promise((resolve) => setTimeout(resolve, ms)); 646 | }; 647 | 648 | module.exports.downloadLogs = async (vodId, app, cursor = null, retry = 1) => { 649 | let comments = [], 650 | response, 651 | lastCursor; 652 | 653 | if (!cursor) { 654 | let offset = 0; 655 | await app 656 | .service("logs") 657 | .find({ 658 | paginate: false, 659 | query: { 660 | vod_id: vodId, 661 | }, 662 | }) 663 | .then((data) => { 664 | if (data.length > 0) 665 | offset = parseFloat(data[data.length - 1].content_offset_seconds); 666 | }) 667 | .catch((e) => { 668 | console.error(e); 669 | }); 670 | response = await twitch.fetchComments(vodId, offset); 671 | if (!response?.comments) { 672 | console.info(`No comments for vod ${vodId} at offset ${offset}`); 673 | app.set(`${config.channel}-${vodId}-chat-downloading`, false); 674 | return; 675 | } 676 | let responseComments = response.comments.edges; 677 | 678 | for (let comment of responseComments) { 679 | cursor = comment.cursor; 680 | let node = comment.node; 681 | if (await commentExists(node.id, app)) continue; 682 | const commenter = node.commenter; 683 | const message = node.message; 684 | comments.push({ 685 | id: node.id, 686 | vod_id: vodId, 687 | display_name: commenter ? commenter.displayName : null, 688 | content_offset_seconds: node.contentOffsetSeconds, 689 | message: message.fragments, 690 | user_badges: message.userBadges, 691 | user_color: message.userColor, 692 | createdAt: node.createdAt, 693 | }); 694 | } 695 | } 696 | 697 | while (cursor) { 698 | lastCursor = cursor; 699 | response = await twitch.fetchNextComments(vodId, cursor); 700 | if (!response?.comments) { 701 | console.info( 702 | `No more comments left due to vod ${vodId} being deleted or errored out..` 703 | ); 704 | break; 705 | } 706 | 707 | responseComments = response.comments.edges; 708 | 709 | if ((process.env.NODE_ENV || "").trim() !== "production") { 710 | readline.clearLine(process.stdout, 0); 711 | readline.cursorTo(process.stdout, 0, null); 712 | process.stdout.write( 713 | `Current Log position: ${dayjs 714 | .duration(responseComments[0].node.contentOffsetSeconds, "s") 715 | .format("HH:mm:ss")}` 716 | ); 717 | } 718 | 719 | for (let comment of responseComments) { 720 | cursor = comment.cursor; 721 | let node = comment.node; 722 | if (await commentExists(node.id, app)) continue; 723 | const commenter = node.commenter; 724 | const message = node.message; 725 | 726 | if (comments.length >= 2500) { 727 | await app 728 | .service("logs") 729 | .create(comments) 730 | .then(() => { 731 | if ((process.env.NODE_ENV || "").trim() !== "production") { 732 | console.info( 733 | `\nSaved ${comments.length} comments in DB for vod ${vodId}` 734 | ); 735 | } 736 | }) 737 | .catch((e) => { 738 | console.error(e); 739 | }); 740 | comments = []; 741 | } 742 | 743 | comments.push({ 744 | id: node.id, 745 | vod_id: vodId, 746 | display_name: commenter ? commenter.displayName : null, 747 | content_offset_seconds: node.contentOffsetSeconds, 748 | message: message.fragments, 749 | user_badges: message.userBadges, 750 | user_color: message.userColor, 751 | createdAt: node.createdAt, 752 | }); 753 | } 754 | 755 | await sleep(150); //don't bombarade the api 756 | } 757 | 758 | if (comments.length > 0) { 759 | await app 760 | .service("logs") 761 | .create(comments) 762 | .then(() => { 763 | if ((process.env.NODE_ENV || "").trim() !== "production") { 764 | console.info( 765 | `Finished current log position: ${ 766 | responseComments[responseComments.length - 1].node 767 | .contentOffsetSeconds * 1000 768 | }` 769 | ); 770 | } 771 | }) 772 | .catch((e) => { 773 | console.error(e); 774 | }); 775 | } 776 | 777 | //if live, continue fetching logs. 778 | const stream = await twitch.getStream(config.twitch.id); 779 | 780 | if (stream && stream[0]) { 781 | setTimeout(() => { 782 | this.downloadLogs(vodId, app, lastCursor); 783 | }, 1000 * 60 * 1); 784 | //retry for next 10 mins if not live anymore to catch remaining logs. 785 | } else if (retry < 10) { 786 | retry++; 787 | setTimeout(() => { 788 | this.downloadLogs(vodId, app, lastCursor, retry); 789 | }, 1000 * 60 * 1); 790 | } else { 791 | console.info(`Saved all comments in DB for vod ${vodId}`); 792 | app.set(`${config.channel}-${vodId}-chat-downloading`, false); 793 | emotes.save(vodId, app); 794 | } 795 | }; 796 | 797 | //RETRY PARAM: Just to make sure whole vod is processed bc it takes awhile for twitch to update the vod even after a stream ends. 798 | //VOD TS FILES SEEMS TO UPDATE AROUND 5 MINUTES. DELAY IS TO CHECK EVERY X MIN. 799 | module.exports.download = async ( 800 | vodId, 801 | app, 802 | retry = 0, 803 | delay = 1, 804 | liveDownload = false 805 | ) => { 806 | if ((process.env.NODE_ENV || "").trim() !== "production") 807 | console.info(`${vodId} Download Retry: ${retry}`); 808 | const dir = `${config.vodPath}/${vodId}`; 809 | const m3u8Path = `${dir}/${vodId}.m3u8`; 810 | const newVodData = await twitch.getVodData(vodId); 811 | const m3u8Exists = await fileExists(m3u8Path); 812 | let duration, vod; 813 | await app 814 | .service("vods") 815 | .get(vodId) 816 | .then((data) => { 817 | vod = data; 818 | }) 819 | .catch(() => {}); 820 | 821 | if (!vod) 822 | return console.error("Failed to download video: no VOD in database"); 823 | 824 | if (m3u8Exists) { 825 | if (newVodData) await this.saveChapters(vodId, app, duration); 826 | } 827 | 828 | if ( 829 | duration >= config.youtube.splitDuration && 830 | config.youtube.liveUpload && 831 | config.youtube.upload 832 | ) { 833 | const noOfParts = Math.floor(duration / config.youtube.splitDuration); 834 | 835 | const vod_youtube_data = vod.youtube.filter((data) => { 836 | return data.type === "vod"; 837 | }); 838 | if (vod_youtube_data.length < noOfParts) { 839 | for (let i = 0; i < noOfParts; i++) { 840 | if (vod_youtube_data[i]) continue; 841 | await this.liveUploadPart( 842 | app, 843 | vodId, 844 | m3u8Path, 845 | config.youtube.splitDuration * i, 846 | config.youtube.splitDuration, 847 | i + 1 848 | ); 849 | } 850 | } 851 | } 852 | 853 | if ((!newVodData && m3u8Exists) || retry >= 10) { 854 | app.set(`${config.channel}-${vodId}-vod-downloading`, false); 855 | 856 | const mp4Path = `${config.vodPath}/${vodId}.mp4`; 857 | await this.convertToMp4(m3u8Path, vodId, mp4Path); 858 | if (config.drive.upload) await drive.upload(vodId, mp4Path, app); 859 | if (config.youtube.liveUpload && config.youtube.upload) { 860 | //upload last part 861 | let startTime = 0; 862 | 863 | const vod_youtube_data = vod.youtube.filter( 864 | (data) => data.type === "vod" 865 | ); 866 | for (let i = 0; i < vod_youtube_data.length; i++) { 867 | startTime += vod_youtube_data[i].duration; 868 | } 869 | await this.liveUploadPart( 870 | app, 871 | vodId, 872 | m3u8Path, 873 | startTime, 874 | duration - startTime, 875 | vod_youtube_data.length + 1 876 | ); 877 | //save parts at last upload. 878 | setTimeout(() => youtube.saveParts(vodId, app, "vod"), 60000); 879 | } else if (config.youtube.upload) { 880 | await this.upload(vodId, app, mp4Path); 881 | if (!config.saveMP4) await fs.promises.rm(mp4Path); 882 | } 883 | if (!config.saveHLS) 884 | await fs.promises.rm(dir, { 885 | recursive: true, 886 | }); 887 | return; 888 | } 889 | 890 | const tokenSig = await twitch.getVodTokenSig(vodId); 891 | if (!tokenSig) { 892 | setTimeout(() => { 893 | this.download(vodId, app, retry, delay, liveDownload); 894 | }, 1000 * 60 * delay); 895 | return console.error(`failed to get token/sig for ${vodId}`); 896 | } 897 | 898 | let newVideoM3u8 = await twitch.getM3u8( 899 | vodId, 900 | tokenSig.value, 901 | tokenSig.signature 902 | ); 903 | if (!newVideoM3u8) { 904 | setTimeout(() => { 905 | this.download(vodId, app, retry, delay, liveDownload); 906 | }, 1000 * 60 * delay); 907 | return console.error("failed to get m3u8"); 908 | } 909 | 910 | let parsedM3u8 = twitch.getParsedM3u8(newVideoM3u8); 911 | if (!parsedM3u8) { 912 | setTimeout(() => { 913 | this.download(vodId, app, retry, delay, liveDownload); 914 | }, 1000 * 60 * delay); 915 | console.error(newVideoM3u8); 916 | return console.error("failed to parse m3u8"); 917 | } 918 | 919 | const baseURL = parsedM3u8.substring(0, parsedM3u8.lastIndexOf("/")); 920 | 921 | let variantM3u8 = await twitch.getVariantM3u8(parsedM3u8); 922 | if (!variantM3u8) { 923 | setTimeout(() => { 924 | this.download(vodId, app, retry, delay, liveDownload); 925 | }, 1000 * 60 * delay); 926 | return console.error("failed to get variant m3u8"); 927 | } 928 | 929 | //Save duration 930 | duration = await hlsGetDuration(variantM3u8); 931 | await saveDuration(vodId, duration, app); 932 | 933 | variantM3u8 = HLS.parse(variantM3u8); 934 | if (liveDownload) variantM3u8 = checkForUnmutedTS(variantM3u8); 935 | 936 | if (!(await fileExists(m3u8Path))) { 937 | if (!(await fileExists(dir))) { 938 | fs.mkdirSync(dir); 939 | } 940 | await downloadTSFiles(variantM3u8, dir, baseURL, vodId); 941 | 942 | setTimeout(() => { 943 | this.download(vodId, app, retry, delay, liveDownload); 944 | }, 1000 * 60 * delay); 945 | return; 946 | } 947 | 948 | let videoM3u8 = await fs.promises.readFile(m3u8Path, "utf8").catch((e) => { 949 | console.error(e); 950 | return null; 951 | }); 952 | 953 | if (!videoM3u8) { 954 | setTimeout(() => { 955 | this.download(vodId, app, retry, delay, liveDownload); 956 | }, 1000 * 60 * delay); 957 | return; 958 | } 959 | 960 | videoM3u8 = HLS.parse(videoM3u8); 961 | 962 | //retry if last segment is the same as on file m3u8 and if the actual segment exists. 963 | if ( 964 | variantM3u8.segments[variantM3u8.segments.length - 1].uri === 965 | videoM3u8.segments[videoM3u8.segments.length - 1].uri && 966 | (await fileExists( 967 | `${dir}/${variantM3u8.segments[variantM3u8.segments.length - 1].uri}` 968 | )) 969 | ) { 970 | retry++; 971 | setTimeout(() => { 972 | this.download(vodId, app, retry, delay, liveDownload); 973 | }, 1000 * 60 * delay); 974 | return; 975 | } 976 | 977 | //reset retry if downloading new ts files. 978 | retry = 1; 979 | await downloadTSFiles(variantM3u8, dir, baseURL, vodId); 980 | 981 | setTimeout(() => { 982 | this.download(vodId, app, retry, delay, liveDownload); 983 | }, 1000 * 60 * delay); 984 | }; 985 | 986 | const checkForUnmutedTS = (m3u8) => { 987 | for (let i = 0; i < m3u8.segments.length; i++) { 988 | const segment = m3u8.segments[i]; 989 | if (segment.uri.includes("-muted")) { 990 | m3u8.segments[i].uri = `${segment.uri.substring( 991 | 0, 992 | segment.uri.indexOf("-muted") 993 | )}.ts`; 994 | continue; 995 | } 996 | if (segment.uri.includes("-unmuted")) { 997 | m3u8.segments[i].uri = `${segment.uri.substring( 998 | 0, 999 | segment.uri.indexOf("-unmuted") 1000 | )}.ts`; 1001 | } 1002 | } 1003 | return m3u8; 1004 | }; 1005 | 1006 | const downloadTSFiles = async (m3u8, dir, baseURL, vodId) => { 1007 | try { 1008 | fs.writeFileSync(`${dir}/${vodId}.m3u8`, HLS.stringify(m3u8)); 1009 | } catch (err) { 1010 | console.error(err); 1011 | } 1012 | for (let segment of m3u8.segments) { 1013 | if (await fileExists(`${dir}/${segment.uri}`)) continue; 1014 | 1015 | await axios({ 1016 | method: "get", 1017 | url: `${baseURL}/${segment.uri}`, 1018 | responseType: "stream", 1019 | }) 1020 | .then((response) => { 1021 | if ((process.env.NODE_ENV || "").trim() !== "production") { 1022 | console.info(`Downloaded ${segment.uri}`); 1023 | } 1024 | response.data.pipe(fs.createWriteStream(`${dir}/${segment.uri}`)); 1025 | }) 1026 | .catch((e) => { 1027 | console.error(e); 1028 | }); 1029 | } 1030 | if ((process.env.NODE_ENV || "").trim() !== "production") { 1031 | console.info( 1032 | `Done downloading.. Last segment was ${ 1033 | m3u8.segments[m3u8.segments.length - 1].uri 1034 | }` 1035 | ); 1036 | } 1037 | }; 1038 | 1039 | module.exports.convertToMp4 = async (m3u8, vodId, mp4Path) => { 1040 | await new Promise((resolve, reject) => { 1041 | const ffmpeg_process = ffmpeg(m3u8); 1042 | ffmpeg_process 1043 | .videoCodec("copy") 1044 | .audioCodec("copy") 1045 | .outputOptions(["-bsf:a aac_adtstoasc"]) 1046 | .toFormat("mp4") 1047 | .on("progress", (progress) => { 1048 | if ((process.env.NODE_ENV || "").trim() !== "production") { 1049 | readline.clearLine(process.stdout, 0); 1050 | readline.cursorTo(process.stdout, 0, null); 1051 | process.stdout.write( 1052 | `M3U8 CONVERT TO MP4 PROGRESS: ${Math.round(progress.percent)}%` 1053 | ); 1054 | } 1055 | }) 1056 | .on("start", (cmd) => { 1057 | if ((process.env.NODE_ENV || "").trim() !== "production") { 1058 | console.info(cmd); 1059 | } 1060 | console.info(`Converting ${vodId} m3u8 to mp4`); 1061 | }) 1062 | .on("error", function (err) { 1063 | ffmpeg_process.kill("SIGKILL"); 1064 | reject(err); 1065 | }) 1066 | .on("end", function () { 1067 | resolve(); 1068 | }) 1069 | .saveToFile(mp4Path); 1070 | }); 1071 | }; 1072 | 1073 | const fileExists = async (file) => { 1074 | return fs.promises 1075 | .access(file, fs.constants.F_OK) 1076 | .then(() => true) 1077 | .catch(() => false); 1078 | }; 1079 | 1080 | //EXT-X-TWITCH-TOTAL-SECS use this to get total duration from m3u8 1081 | const hlsGetDuration = async (m3u8) => { 1082 | let totalSeconds; 1083 | for (let line of m3u8.split("\n")) { 1084 | if (!line.startsWith("#EXT-X-TWITCH-TOTAL-SECS:")) continue; 1085 | const split = line.split(":"); 1086 | if (split[1]) totalSeconds = parseInt(split[1]); 1087 | break; 1088 | } 1089 | return totalSeconds; 1090 | }; 1091 | 1092 | const getDuration = async (video) => { 1093 | let duration; 1094 | await new Promise((resolve, reject) => { 1095 | ffmpeg.ffprobe(video, (err, metadata) => { 1096 | if (err) { 1097 | console.error(err); 1098 | return reject(); 1099 | } 1100 | duration = metadata.format.duration; 1101 | resolve(); 1102 | }); 1103 | }); 1104 | return Math.round(duration); 1105 | }; 1106 | 1107 | const saveDuration = async (vodId, duration, app) => { 1108 | if (isNaN(duration)) return; 1109 | duration = toHHMMSS(duration); 1110 | 1111 | await app 1112 | .service("vods") 1113 | .patch(vodId, { 1114 | duration: duration, 1115 | }) 1116 | .catch((e) => { 1117 | console.error(e); 1118 | }); 1119 | }; 1120 | 1121 | const toHHMMSS = (secs) => { 1122 | var sec_num = parseInt(secs, 10); 1123 | var hours = Math.floor(sec_num / 3600); 1124 | var minutes = Math.floor(sec_num / 60) % 60; 1125 | var seconds = sec_num % 60; 1126 | 1127 | return [hours, minutes, seconds] 1128 | .map((v) => (v < 10 ? "0" + v : v)) 1129 | .filter((v, i) => v !== "00" || i > 0) 1130 | .join(":"); 1131 | }; 1132 | 1133 | module.exports.saveChapters = async (vodId, app, duration) => { 1134 | const chapters = await twitch.getChapters(vodId); 1135 | if (!chapters) 1136 | return console.error("Failed to save chapters: Chapters is null"); 1137 | 1138 | let newChapters = []; 1139 | if (chapters.length === 0) { 1140 | const chapter = await twitch.getChapter(vodId); 1141 | if (!chapter) return null; 1142 | const gameData = chapter.game 1143 | ? await twitch.getGameData(chapter.game.id) 1144 | : null; 1145 | newChapters.push({ 1146 | gameId: chapter.game ? chapter.game.id : null, 1147 | name: chapter.game ? chapter.game.displayName : null, 1148 | image: gameData 1149 | ? gameData.box_art_url.replace("{width}x{height}", "40x53") 1150 | : null, 1151 | duration: "00:00:00", 1152 | start: 0, 1153 | end: duration, 1154 | }); 1155 | } else { 1156 | for (let chapter of chapters) { 1157 | newChapters.push({ 1158 | gameId: chapter.node.details.game ? chapter.node.details.game.id : null, 1159 | name: chapter.node.details.game 1160 | ? chapter.node.details.game.displayName 1161 | : null, 1162 | image: chapter.node.details.game 1163 | ? chapter.node.details.game.boxArtURL 1164 | : null, 1165 | duration: dayjs 1166 | .duration(chapter.node.positionMilliseconds, "ms") 1167 | .format("HH:mm:ss"), 1168 | start: 1169 | chapter.node.positionMilliseconds === 0 1170 | ? chapter.node.positionMilliseconds / 1000 1171 | : chapter.node.positionMilliseconds / 1000, 1172 | end: 1173 | chapter.node.durationMilliseconds === 0 1174 | ? duration - chapter.node.positionMilliseconds / 1000 1175 | : chapter.node.durationMilliseconds / 1000, 1176 | }); 1177 | } 1178 | } 1179 | 1180 | await app 1181 | .service("vods") 1182 | .patch(vodId, { 1183 | chapters: newChapters, 1184 | }) 1185 | .catch((e) => { 1186 | console.error(e); 1187 | }); 1188 | }; 1189 | 1190 | module.exports.getLogs = async (vodId, app) => { 1191 | console.info(`Saving logs for ${vodId}`); 1192 | let start_time = new Date(); 1193 | let comments = []; 1194 | let cursor; 1195 | let response = await twitch.fetchComments(vodId); 1196 | if (!response?.comments) { 1197 | console.info(`No comments for vod ${vodId}`); 1198 | return; 1199 | } 1200 | let responseComments = response.comments.edges; 1201 | 1202 | for (let comment of responseComments) { 1203 | cursor = comment.cursor; 1204 | let node = comment.node; 1205 | if (await commentExists(node.id, app)) continue; 1206 | const commenter = node.commenter; 1207 | const message = node.message; 1208 | comments.push({ 1209 | id: node.id, 1210 | vod_id: vodId, 1211 | display_name: commenter ? commenter.displayName : null, 1212 | content_offset_seconds: node.contentOffsetSeconds, 1213 | message: message.fragments, 1214 | user_badges: message.userBadges, 1215 | user_color: message.userColor, 1216 | createdAt: node.createdAt, 1217 | }); 1218 | } 1219 | 1220 | let howMany = 1; 1221 | while (cursor) { 1222 | response = await twitch.fetchNextComments(vodId, cursor); 1223 | if (!response?.comments) { 1224 | console.info( 1225 | `No more comments left due to vod ${vodId} being deleted or errored out..` 1226 | ); 1227 | break; 1228 | } 1229 | 1230 | responseComments = response.comments.edges; 1231 | 1232 | if ((process.env.NODE_ENV || "").trim() !== "production") { 1233 | readline.clearLine(process.stdout, 0); 1234 | readline.cursorTo(process.stdout, 0, null); 1235 | process.stdout.write( 1236 | `Current Log position: ${dayjs 1237 | .duration(responseComments[0].node.contentOffsetSeconds, "s") 1238 | .format("HH:mm:ss")}` 1239 | ); 1240 | } 1241 | 1242 | for (let comment of responseComments) { 1243 | cursor = comment.cursor; 1244 | let node = comment.node; 1245 | if (await commentExists(node.id, app)) continue; 1246 | const commenter = node.commenter; 1247 | const message = node.message; 1248 | 1249 | if (comments.length >= 2500) { 1250 | await app 1251 | .service("logs") 1252 | .create(comments) 1253 | .then(() => { 1254 | if ((process.env.NODE_ENV || "").trim() !== "production") { 1255 | console.info( 1256 | `\nSaved ${comments.length} comments in DB for vod ${vodId}` 1257 | ); 1258 | } 1259 | }) 1260 | .catch((e) => { 1261 | console.error(e); 1262 | }); 1263 | comments = []; 1264 | } 1265 | 1266 | comments.push({ 1267 | id: node.id, 1268 | vod_id: vodId, 1269 | display_name: commenter ? commenter.displayName : null, 1270 | content_offset_seconds: node.contentOffsetSeconds, 1271 | message: message.fragments, 1272 | user_badges: message.userBadges, 1273 | user_color: message.userColor, 1274 | createdAt: node.createdAt, 1275 | }); 1276 | } 1277 | 1278 | await sleep(150); //don't bombarade the api 1279 | 1280 | howMany++; 1281 | } 1282 | console.info( 1283 | `\nTotal API Calls: ${howMany} | Total Time to get logs for ${vodId}: ${ 1284 | (new Date() - start_time) / 1000 1285 | } seconds` 1286 | ); 1287 | 1288 | await app 1289 | .service("logs") 1290 | .create(comments) 1291 | .then(() => { 1292 | console.info(`Saved all comments in DB for vod ${vodId}`); 1293 | }) 1294 | .catch(() => {}); 1295 | }; 1296 | 1297 | module.exports.manualLogs = async (commentsPath, vodId, app) => { 1298 | let start_time = new Date(), 1299 | comments = [], 1300 | responseComments, 1301 | howMany = 1; 1302 | await fs.promises 1303 | .readFile(commentsPath) 1304 | .then((data) => { 1305 | responseComments = JSON.parse(data).comments.edges; 1306 | }) 1307 | .catch((e) => { 1308 | console.error(e); 1309 | }); 1310 | 1311 | for (let comment of responseComments) { 1312 | let node = comment.node; 1313 | if ((process.env.NODE_ENV || "").trim() !== "production") { 1314 | readline.clearLine(process.stdout, 0); 1315 | readline.cursorTo(process.stdout, 0, null); 1316 | process.stdout.write( 1317 | `Current Log position: ${dayjs 1318 | .duration(node.contentOffsetSeconds, "s") 1319 | .format("HH:mm:ss")}` 1320 | ); 1321 | } 1322 | if (await commentExists(node.id, app)) continue; 1323 | 1324 | if (comments.length >= 2500) { 1325 | await app 1326 | .service("logs") 1327 | .create(comments) 1328 | .then(() => { 1329 | if ((process.env.NODE_ENV || "").trim() !== "production") { 1330 | console.info( 1331 | `\nSaved ${comments.length} comments in DB for vod ${vodId}` 1332 | ); 1333 | } 1334 | }) 1335 | .catch((e) => { 1336 | console.error(e); 1337 | }); 1338 | comments = []; 1339 | } 1340 | 1341 | const commenter = node.commenter; 1342 | const message = node.message; 1343 | 1344 | comments.push({ 1345 | id: node.id, 1346 | vod_id: vodId, 1347 | display_name: commenter ? commenter.displayName : null, 1348 | content_offset_seconds: node.contentOffsetSeconds, 1349 | message: message.fragments, 1350 | user_badges: message.userBadges, 1351 | user_color: message.userColor, 1352 | createdAt: node.createdAt, 1353 | }); 1354 | 1355 | howMany++; 1356 | } 1357 | console.info( 1358 | `\nTotal Comments: ${howMany} | Total Time to get logs for ${vodId}: ${ 1359 | (new Date() - start_time) / 1000 1360 | } seconds` 1361 | ); 1362 | 1363 | await app 1364 | .service("logs") 1365 | .create(comments) 1366 | .then(() => { 1367 | console.info(`Saved all comments in DB for vod ${vodId}`); 1368 | }) 1369 | .catch(() => {}); 1370 | }; 1371 | 1372 | module.exports.mp4Download = async (vodId) => { 1373 | const tokenSig = await twitch.getVodTokenSig(vodId); 1374 | if (!tokenSig) return console.error(`failed to get token/sig for ${vodId}`); 1375 | 1376 | let m3u8 = await twitch.getM3u8(vodId, tokenSig.value, tokenSig.signature); 1377 | if (!m3u8) return null; 1378 | 1379 | m3u8 = twitch.getParsedM3u8(m3u8); 1380 | if (!m3u8) return null; 1381 | 1382 | const vodPath = `${config.vodPath}/${vodId}.mp4`; 1383 | 1384 | const success = await this.ffmpegMp4Download(m3u8, vodPath) 1385 | .then(() => { 1386 | console.info(`Downloaded ${vodId}.mp4\n`); 1387 | return true; 1388 | }) 1389 | .catch((e) => { 1390 | console.error("\nffmpeg error occurred: " + e); 1391 | return false; 1392 | }); 1393 | 1394 | if (success) return vodPath; 1395 | 1396 | return null; 1397 | }; 1398 | 1399 | module.exports.ffmpegMp4Download = async (m3u8, path) => { 1400 | return new Promise((resolve, reject) => { 1401 | const ffmpeg_process = ffmpeg(m3u8); 1402 | ffmpeg_process 1403 | .videoCodec("copy") 1404 | .audioCodec("copy") 1405 | .outputOptions(["-bsf:a aac_adtstoasc", "-copyts", "-start_at_zero"]) 1406 | .toFormat("mp4") 1407 | .on("progress", (progress) => { 1408 | if ((process.env.NODE_ENV || "").trim() !== "production") { 1409 | readline.clearLine(process.stdout, 0); 1410 | readline.cursorTo(process.stdout, 0, null); 1411 | process.stdout.write( 1412 | `DOWNLOAD PROGRESS: ${Math.round(progress.percent)}%` 1413 | ); 1414 | } 1415 | }) 1416 | .on("start", (cmd) => { 1417 | console.info(`Starting m3u8 download for ${m3u8} in ${path}`); 1418 | }) 1419 | .on("error", function (err) { 1420 | ffmpeg_process.kill("SIGKILL"); 1421 | reject(err); 1422 | }) 1423 | .on("end", function () { 1424 | resolve(); 1425 | }) 1426 | .saveToFile(path); 1427 | }); 1428 | }; 1429 | -------------------------------------------------------------------------------- /src/middleware/youtube.js: -------------------------------------------------------------------------------- 1 | const config = require("../../config/config.json"); 2 | const fs = require("fs"); 3 | const { google } = require("googleapis"); 4 | const readline = require("readline"); 5 | const dayjs = require("dayjs"); 6 | const duration = require("dayjs/plugin/duration"); 7 | dayjs.extend(duration); 8 | 9 | module.exports.chapters = function (app) { 10 | const _this = this; 11 | return async function (req, res, next) { 12 | if (!req.body.vodId) 13 | return res.status(400).json({ error: true, message: "Missing vod id.." }); 14 | 15 | if (!req.body.type) 16 | return res 17 | .status(400) 18 | .json({ error: true, message: "Missing type param..." }); 19 | 20 | _this.saveChapters(req.body.vodId, app, req.body.type); 21 | 22 | res 23 | .status(200) 24 | .json({ error: false, message: `Saving chapters for ${req.body.vodId}` }); 25 | }; 26 | }; 27 | 28 | module.exports.saveChapters = async (vodId, app, type = "vod") => { 29 | const oauth2Client = app.get("ytOauth2Client"); 30 | const youtube = google.youtube({ 31 | version: "v3", 32 | auth: oauth2Client, 33 | }); 34 | await youtube.search.list({ 35 | auth: oauth2Client, 36 | part: "id,snippet", 37 | q: "Check if token is valid", 38 | }); 39 | await sleep(5000); 40 | 41 | let vod_data; 42 | await app 43 | .service("vods") 44 | .get(vodId) 45 | .then((data) => { 46 | vod_data = data; 47 | }) 48 | .catch(() => {}); 49 | 50 | if (!vod_data) 51 | return console.error(`Could not save chapters: Can't find vod ${vodId}..`); 52 | 53 | if (!vod_data.chapters) 54 | return console.error( 55 | `Could not save chapters: Can't find chapters for vod ${vodId}..` 56 | ); 57 | 58 | console.log(`Saving chapters on youtube for ${vodId}`); 59 | 60 | const type_youtube_data = vod_data.youtube.filter( 61 | (data) => data.type === type 62 | ); 63 | for (let youtube_data of type_youtube_data) { 64 | const video_data = await this.getVideo(youtube_data.id, oauth2Client); 65 | if (!video_data) 66 | return console.error( 67 | `Could not save chapters: Can't find ${youtube_data.id} youtube video..` 68 | ); 69 | const snippet = video_data.snippet; 70 | 71 | let description = (snippet.description += "\n\n"); 72 | for (let chapter of vod_data.chapters) { 73 | const startDuration = 74 | config.youtube.splitDuration * (youtube_data.part - 1); 75 | const endDuration = config.youtube.splitDuration * youtube_data.part; 76 | 77 | if ( 78 | chapter.start <= endDuration && 79 | chapter.start + chapter.end >= startDuration 80 | ) { 81 | const actualTime = chapter.start - startDuration; 82 | const timestamp = actualTime < 0 ? 0 : actualTime; 83 | description += `${dayjs.duration(timestamp, "s").format("HH:mm:ss")} ${ 84 | chapter.name 85 | }\n`; 86 | } 87 | 88 | const res = await youtube.videos.update({ 89 | resource: { 90 | id: youtube_data.id, 91 | snippet: { 92 | title: snippet.title, 93 | description: description, 94 | categoryId: snippet.categoryId, 95 | }, 96 | }, 97 | part: "snippet", 98 | }); 99 | } 100 | } 101 | }; 102 | 103 | module.exports.parts = function (app) { 104 | const _this = this; 105 | return async function (req, res, next) { 106 | if (!req.body.vodId) 107 | return res.status(400).json({ error: true, message: "Missing vod id.." }); 108 | 109 | if (!req.body.type) 110 | return res 111 | .status(400) 112 | .json({ error: true, message: "Missing Type param..." }); 113 | 114 | _this.saveParts(req.body.vodId, app, req.body.type); 115 | 116 | res 117 | .status(200) 118 | .json({ error: false, message: `Saving Parts for ${req.body.vodId}` }); 119 | }; 120 | }; 121 | 122 | module.exports.saveParts = async (vodId, app, type = "vod") => { 123 | const oauth2Client = app.get("ytOauth2Client"); 124 | const youtube = google.youtube({ 125 | version: "v3", 126 | auth: oauth2Client, 127 | }); 128 | let vod_data; 129 | await app 130 | .service("vods") 131 | .get(vodId) 132 | .then((data) => { 133 | vod_data = data; 134 | }) 135 | .catch(() => {}); 136 | 137 | if (!vod_data) 138 | return console.error(`Could not save parts: Can't find vod ${vodId}..`); 139 | 140 | if (vod_data.youtube.length <= 1) 141 | return console.error( 142 | `Could not save parts: (No or only one) youtube video for ${vodId}` 143 | ); 144 | 145 | console.log(`Saving parts on youtube for ${vodId}`); 146 | 147 | const type_youtube_data = vod_data.youtube.filter(function (data) { 148 | return data.type === type; 149 | }); 150 | for (let youtube_data of type_youtube_data) { 151 | const video_data = await this.getVideo(youtube_data.id, oauth2Client); 152 | const snippet = video_data.snippet; 153 | 154 | let description = ``; 155 | for (let i = 0; i < type_youtube_data.length; i++) { 156 | if (youtube_data.id === type_youtube_data[i].id) continue; 157 | description += `PART ${i + 1}: https://youtube.com/watch?v=${ 158 | type_youtube_data[i].id 159 | }\n`; 160 | } 161 | description += "\n" + snippet.description; 162 | 163 | const res = await youtube.videos.update({ 164 | resource: { 165 | id: youtube_data.id, 166 | snippet: { 167 | title: snippet.title, 168 | description: description, 169 | categoryId: snippet.categoryId, 170 | }, 171 | }, 172 | part: "snippet", 173 | }); 174 | 175 | //console.info(res.data); 176 | } 177 | }; 178 | 179 | module.exports.getVideo = async (id, oauth2Client) => { 180 | const youtube = google.youtube({ 181 | version: "v3", 182 | auth: oauth2Client, 183 | }); 184 | const response = await youtube.videos.list({ 185 | part: "contentDetails,snippet", 186 | id: [id], 187 | }); 188 | 189 | const item = response.data.items[0]; 190 | if (!item) return null; 191 | 192 | return item; 193 | }; 194 | 195 | const sleep = (ms) => { 196 | return new Promise((resolve) => setTimeout(resolve, ms)); 197 | }; 198 | 199 | module.exports.upload = async (data, app, isVod = true) => { 200 | const oauth2Client = app.get("ytOauth2Client"); 201 | const youtube = google.youtube("v3"); 202 | 203 | await youtube.search.list({ 204 | auth: oauth2Client, 205 | part: "id,snippet", 206 | q: "Check if token is valid", 207 | }); 208 | await sleep(5000); 209 | 210 | return new Promise(async (resolve, reject) => { 211 | const fileSize = fs.statSync(data.path).size; 212 | const vodTitle = data.vod.title.replace(/>| { 239 | if ((process.env.NODE_ENV || "").trim() !== "production") { 240 | const progress = (evt.bytesRead / fileSize) * 100; 241 | readline.clearLine(process.stdout, 0); 242 | readline.cursorTo(process.stdout, 0, null); 243 | process.stdout.write(`UPLOAD PROGRESS: ${Math.round(progress)}%`); 244 | } 245 | }, 246 | } 247 | ); 248 | if ((process.env.NODE_ENV || "").trim() !== "production") { 249 | console.info("\n\n"); 250 | } 251 | 252 | console.info( 253 | isVod 254 | ? `Uploaded ${data.vod.id} ${data.part} ${data.type} to youtube!` 255 | : `Uploaded ${data.title} to youtube!` 256 | ); 257 | 258 | if (isVod) { 259 | let vod_youtube; 260 | await app 261 | .service("vods") 262 | .get(data.vod.id) 263 | .then((newData) => { 264 | vod_youtube = newData.youtube; 265 | }) 266 | .catch((e) => { 267 | console.error(e); 268 | }); 269 | 270 | if (!vod_youtube) return console.error("Could not find youtube data..."); 271 | 272 | let videoIndex; 273 | for (let i = 0; i < vod_youtube.length; i++) { 274 | const youtube_data = vod_youtube[i]; 275 | if (data.type !== youtube_data.type) continue; 276 | if (data.part === parseInt(youtube_data.part)) { 277 | videoIndex = i; 278 | break; 279 | } 280 | } 281 | 282 | if (videoIndex == undefined) { 283 | vod_youtube.push({ 284 | id: res.data.id, 285 | type: data.type, 286 | duration: data.duration, 287 | part: data.part, 288 | thumbnail_url: res.data.snippet.thumbnails.medium.url, 289 | }); 290 | } else { 291 | vod_youtube[videoIndex] = { 292 | id: res.data.id, 293 | type: data.type, 294 | duration: data.duration, 295 | part: data.part, 296 | thumbnail_url: res.data.snippet.thumbnails.medium.url, 297 | }; 298 | } 299 | 300 | await app 301 | .service("vods") 302 | .patch(data.vod.id, { 303 | youtube: vod_youtube, 304 | thumbnail_url: res.data.snippet.thumbnails.medium.url, 305 | }) 306 | .then(() => { 307 | console.info( 308 | `Saved youtube data in DB for vod ${data.vod.id} ${data.type}` 309 | ); 310 | }) 311 | .catch((e) => { 312 | console.error(e); 313 | }); 314 | } else { 315 | if (data.gameId) { 316 | await app 317 | .service("games") 318 | .patch(data.gameId, { 319 | video_id: res.data.id, 320 | thumbnail_url: res.data.snippet.thumbnails.medium.url, 321 | }) 322 | .then(() => { 323 | console.info( 324 | `Updated ${data.gameId} - ${data.chapter.name} in games DB for ${data.vod.id}` 325 | ); 326 | }) 327 | .catch((e) => { 328 | console.error(e); 329 | }); 330 | } else { 331 | await app 332 | .service("games") 333 | .create({ 334 | vodId: data.vod.id, 335 | start_time: data.start_time, 336 | end_time: data.end_time, 337 | video_provider: "youtube", 338 | video_id: res.data.id, 339 | thumbnail_url: res.data.snippet.thumbnails.medium.url, 340 | game_id: data.chapter.gameId, 341 | game_name: data.chapter.name, 342 | chapter_image: data.chapter.image, 343 | title: data.gameTitle, 344 | }) 345 | .then(() => { 346 | console.info( 347 | `Created ${data.chapter.name} in games DB for ${data.vod.id}` 348 | ); 349 | }) 350 | .catch((e) => { 351 | console.error(e); 352 | }); 353 | } 354 | } 355 | resolve(); 356 | }); 357 | }; 358 | -------------------------------------------------------------------------------- /src/models/emotes.model.js: -------------------------------------------------------------------------------- 1 | // See https://sequelize.org/master/manual/model-basics.html 2 | // for more of what you can do here. 3 | const Sequelize = require("sequelize"); 4 | const DataTypes = Sequelize.DataTypes; 5 | 6 | module.exports = function (app) { 7 | const sequelizeClient = app.get("sequelizeClient"); 8 | const emotes = sequelizeClient.define( 9 | "emotes", 10 | { 11 | vodId: { 12 | type: DataTypes.TEXT, 13 | primaryKey: true, 14 | allowNull: false, 15 | field: "vod_id", 16 | }, 17 | ffz_emotes: { 18 | type: DataTypes.JSONB, 19 | defaultValue: [], 20 | }, 21 | bttv_emotes: { 22 | type: DataTypes.JSONB, 23 | defaultValue: [], 24 | }, 25 | "7tv_emotes": { 26 | type: DataTypes.JSONB, 27 | defaultValue: [], 28 | }, 29 | }, 30 | { 31 | hooks: { 32 | beforeCount(options) { 33 | options.raw = true; 34 | }, 35 | }, 36 | } 37 | ); 38 | 39 | emotes.associate = function (models) { 40 | emotes.belongsTo(models.vods); 41 | }; 42 | 43 | return emotes; 44 | }; 45 | -------------------------------------------------------------------------------- /src/models/games.model.js: -------------------------------------------------------------------------------- 1 | // See http://docs.sequelizejs.com/en/latest/docs/models-definition/ 2 | // for more of what you can do here. 3 | const Sequelize = require("sequelize"); 4 | const DataTypes = Sequelize.DataTypes; 5 | 6 | module.exports = function (app) { 7 | const sequelizeClient = app.get("sequelizeClient"); 8 | const games = sequelizeClient.define( 9 | "games", 10 | { 11 | id: { 12 | type: DataTypes.BIGINT, 13 | primaryKey: true, 14 | autoIncrement: true, 15 | }, 16 | vodId: { 17 | type: DataTypes.TEXT, 18 | allowNull: false, 19 | field: "vod_id", 20 | }, 21 | start_time: { 22 | type: DataTypes.DECIMAL, 23 | }, 24 | end_time: { 25 | type: DataTypes.DECIMAL, 26 | }, 27 | video_provider: { 28 | type: DataTypes.TEXT, 29 | }, 30 | video_id: { 31 | type: DataTypes.TEXT, 32 | }, 33 | thumbnail_url: { 34 | type: DataTypes.TEXT, 35 | }, 36 | game_id: { 37 | type: DataTypes.TEXT, 38 | }, 39 | game_name: { 40 | type: DataTypes.TEXT, 41 | }, 42 | title: { 43 | type: DataTypes.TEXT, 44 | }, 45 | chapter_image: { 46 | type: DataTypes.TEXT, 47 | }, 48 | }, 49 | { 50 | hooks: { 51 | beforeCount(options) { 52 | options.raw = true; 53 | }, 54 | }, 55 | } 56 | ); 57 | 58 | games.associate = function (models) { 59 | games.belongsTo(models.vods); 60 | }; 61 | 62 | return games; 63 | }; 64 | -------------------------------------------------------------------------------- /src/models/logs.model.js: -------------------------------------------------------------------------------- 1 | // See http://docs.sequelizejs.com/en/latest/docs/models-definition/ 2 | // for more of what you can do here. 3 | const Sequelize = require('sequelize'); 4 | const DataTypes = Sequelize.DataTypes; 5 | 6 | module.exports = function (app) { 7 | const sequelizeClient = app.get('sequelizeClient'); 8 | const logs = sequelizeClient.define('logs', { 9 | id: { 10 | type: DataTypes.UUID, 11 | allowNull: false, 12 | primaryKey: true 13 | }, 14 | _id: { 15 | type: DataTypes.INTEGER, 16 | autoIncrement: true, 17 | }, 18 | vod_id: { 19 | type: DataTypes.TEXT, 20 | allowNull: false 21 | }, 22 | display_name: { 23 | type: DataTypes.TEXT, 24 | allowNull: true 25 | }, 26 | content_offset_seconds: { 27 | type: DataTypes.TEXT, 28 | allowNull: false 29 | }, 30 | message: { 31 | type: DataTypes.JSONB, 32 | allowNull: false 33 | }, 34 | user_badges: { 35 | type: DataTypes.JSONB, 36 | allowNull: false 37 | }, 38 | user_color: { 39 | type: DataTypes.TEXT, 40 | allowNull: false, 41 | }, 42 | }, { 43 | hooks: { 44 | beforeCount(options) { 45 | options.raw = true; 46 | } 47 | } 48 | }); 49 | 50 | // eslint-disable-next-line no-unused-vars 51 | logs.associate = function (models) { 52 | // Define associations here 53 | // See http://docs.sequelizejs.com/en/latest/docs/associations/ 54 | }; 55 | 56 | return logs; 57 | }; 58 | -------------------------------------------------------------------------------- /src/models/streams.model.js: -------------------------------------------------------------------------------- 1 | // See https://sequelize.org/master/manual/model-basics.html 2 | // for more of what you can do here. 3 | const Sequelize = require("sequelize"); 4 | const DataTypes = Sequelize.DataTypes; 5 | 6 | module.exports = function (app) { 7 | const sequelizeClient = app.get("sequelizeClient"); 8 | const streams = sequelizeClient.define( 9 | "streams", 10 | { 11 | id: { 12 | type: DataTypes.BIGINT, 13 | allowNull: false, 14 | primaryKey: true, 15 | }, 16 | started_at: { 17 | type: DataTypes.DATE, 18 | }, 19 | platform: { 20 | type: DataTypes.TEXT, 21 | allowNull: false, 22 | }, 23 | is_live: { 24 | type: DataTypes.BOOLEAN, 25 | }, 26 | }, 27 | { 28 | timestamps: false, 29 | hooks: { 30 | beforeCount(options) { 31 | options.raw = true; 32 | }, 33 | }, 34 | } 35 | ); 36 | 37 | return streams; 38 | }; 39 | -------------------------------------------------------------------------------- /src/models/vods.model.js: -------------------------------------------------------------------------------- 1 | // See http://docs.sequelizejs.com/en/latest/docs/models-definition/ 2 | // for more of what you can do here. 3 | const Sequelize = require("sequelize"); 4 | const DataTypes = Sequelize.DataTypes; 5 | 6 | module.exports = function (app) { 7 | const sequelizeClient = app.get("sequelizeClient"); 8 | const vods = sequelizeClient.define( 9 | "vods", 10 | { 11 | id: { 12 | type: DataTypes.TEXT, 13 | allowNull: false, 14 | primaryKey: true, 15 | }, 16 | chapters: { 17 | type: DataTypes.JSONB, 18 | defaultValue: [], 19 | }, 20 | title: { 21 | type: DataTypes.TEXT, 22 | }, 23 | duration: { 24 | type: DataTypes.TEXT, 25 | defaultValue: "00:00:00", 26 | }, 27 | thumbnail_url: { 28 | type: DataTypes.TEXT, 29 | }, 30 | youtube: { 31 | type: DataTypes.JSONB, 32 | defaultValue: [], 33 | }, 34 | stream_id: { 35 | type: DataTypes.TEXT, 36 | }, 37 | drive: { 38 | type: DataTypes.JSONB, 39 | defaultValue: [], 40 | }, 41 | platform: { 42 | type: DataTypes.TEXT, 43 | allowNull: false, 44 | }, 45 | }, 46 | { 47 | hooks: { 48 | beforeCount(options) { 49 | options.raw = true; 50 | }, 51 | }, 52 | } 53 | ); 54 | 55 | vods.associate = function (models) { 56 | vods.hasOne(models.emotes); 57 | vods.hasMany(models.games); 58 | }; 59 | 60 | return vods; 61 | }; 62 | -------------------------------------------------------------------------------- /src/redis.js: -------------------------------------------------------------------------------- 1 | const { createClient } = require("redis"); 2 | const { RateLimiterRedis } = require("rate-limiter-flexible"); 3 | 4 | module.exports = async function (app) { 5 | const redisConf = app.get("redis"), 6 | client = createClient({ 7 | socket: { 8 | path: redisConf.useSocket ? redisConf.path : null, 9 | host: redisConf.host, 10 | }, 11 | enable_offline_queue: false, 12 | }); 13 | 14 | client.connect().catch((e) => console.error(e)); 15 | 16 | app.set("redisClient", client); 17 | 18 | const rateLimiter = new RateLimiterRedis({ 19 | storeClient: client, 20 | keyPrefix: "middleware", 21 | points: 20, 22 | duration: 5, 23 | useRedisPackage: true, 24 | }); 25 | 26 | app.set("rateLimiter", rateLimiter); 27 | }; 28 | -------------------------------------------------------------------------------- /src/sequelize.js: -------------------------------------------------------------------------------- 1 | const Sequelize = require('sequelize'); 2 | 3 | module.exports = function (app) { 4 | const connectionString = app.get('postgres'); 5 | const sequelize = new Sequelize(connectionString, { 6 | dialect: 'postgres', 7 | logging: false, 8 | define: { 9 | freezeTableName: true 10 | } 11 | }); 12 | const oldSetup = app.setup; 13 | 14 | app.set('sequelizeClient', sequelize); 15 | 16 | app.setup = function (...args) { 17 | const result = oldSetup.apply(this, args); 18 | 19 | // Set up data relationships 20 | const models = sequelize.models; 21 | Object.keys(models).forEach(name => { 22 | if ('associate' in models[name]) { 23 | models[name].associate(models); 24 | } 25 | }); 26 | 27 | // Sync to the database 28 | app.set('sequelizeSync', sequelize.sync()); 29 | 30 | return result; 31 | }; 32 | }; 33 | -------------------------------------------------------------------------------- /src/services/cache.js: -------------------------------------------------------------------------------- 1 | //https://github.com/sarkistlt/feathers-redis-cache/blob/master/src/hooks.ts 2 | const config = require("../../config/config.json"); 3 | const qs = require("qs"); 4 | const dayjs = require("dayjs"); 5 | const duration = require("dayjs/plugin/duration"); 6 | const relativeTime = require("dayjs/plugin/relativeTime"); 7 | dayjs.extend(duration); 8 | dayjs.extend(relativeTime); 9 | 10 | const { DISABLE_REDIS_CACHE, ENABLE_REDIS_CACHE_LOGGER } = process.env; 11 | const HTTP_SERVER_ERROR = 500; 12 | const defaults = { 13 | defaultExpiration: 3600 * 24, // seconds 14 | prefix: config.channel, 15 | }; 16 | 17 | const hashCode = (s) => { 18 | let h; 19 | for (let i = 0; i < s.length; i++) { 20 | h = (Math.imul(31, h) + s.charCodeAt(i)) | 0; 21 | } 22 | return String(h); 23 | }; 24 | 25 | const cacheKey = (hook) => { 26 | const q = hook.params.query || {}; 27 | const p = hook.params.paginate === false ? "disabled" : "enabled"; 28 | let path = `pagination-hook:${p}::${hook.path}`; 29 | 30 | if (hook.id) { 31 | path += `/${hook.id}`; 32 | } 33 | 34 | if (Object.keys(q).length > 0) { 35 | path += `?${qs.stringify(JSON.parse(JSON.stringify(q)), { 36 | encode: false, 37 | })}`; 38 | } 39 | 40 | // {prefix}{group}{key} 41 | return `${hashCode(hook.path)}${hashCode(path)}`; 42 | }; 43 | 44 | module.exports.before = (passedOptions = {}) => { 45 | if (DISABLE_REDIS_CACHE) { 46 | return (hook) => hook; 47 | } 48 | 49 | return function (hook) { 50 | try { 51 | if (hook && hook.params && hook.params.$skipCacheHook) { 52 | return Promise.resolve(hook); 53 | } 54 | 55 | return new Promise(async (resolve) => { 56 | const client = hook.app.get("redisClient"); 57 | const options = { ...defaults, ...passedOptions }; 58 | 59 | if (!client) { 60 | return resolve(hook); 61 | } 62 | 63 | const group = 64 | typeof options.cacheGroupKey === "function" 65 | ? hashCode(`group-${options.cacheGroupKey(hook)}`) 66 | : hashCode(`group-${hook.path || "general"}`); 67 | const path = 68 | typeof options.cacheKey === "function" 69 | ? `${options.prefix}${group}${options.cacheKey(hook)}` 70 | : `${options.prefix}${group}${cacheKey(hook)}`; 71 | 72 | hook.params.cacheKey = path; 73 | 74 | data = await client 75 | .GET(path) 76 | .then((res) => JSON.parse(res)) 77 | .catch(() => resolve(hook)); 78 | 79 | if (!data || !data.expiresOn || !data.cache) { 80 | return resolve(hook); 81 | } 82 | 83 | const duration = dayjs(data.expiresOn).format( 84 | "DD MMMM YYYY - HH:mm:ss" 85 | ); 86 | 87 | hook.result = data.cache; 88 | hook.params.$skipCacheHook = true; 89 | 90 | if (options.env !== "test" && ENABLE_REDIS_CACHE_LOGGER === "true") { 91 | console.log(`[redis] returning cached value for ${path}.`); 92 | console.log(`> Expires on ${duration}.`); 93 | } 94 | 95 | return resolve(hook); 96 | }); 97 | } catch (err) { 98 | console.error(err); 99 | return Promise.resolve(hook); 100 | } 101 | }; 102 | }; 103 | 104 | module.exports.after = (passedOptions = {}) => { 105 | if (DISABLE_REDIS_CACHE) { 106 | return (hook) => hook; 107 | } 108 | 109 | return function (hook) { 110 | try { 111 | if (hook && hook.params && hook.params.$skipCacheHook) { 112 | return Promise.resolve(hook); 113 | } 114 | 115 | if (!hook.result) { 116 | return Promise.resolve(hook); 117 | } 118 | 119 | return new Promise((resolve) => { 120 | const client = hook.app.get("redisClient"); 121 | const options = { ...defaults, ...passedOptions }; 122 | const duration = options.expiration || options.defaultExpiration; 123 | const { cacheKey } = hook.params; 124 | 125 | if (!client || !cacheKey) { 126 | return resolve(hook); 127 | } 128 | 129 | client.SET( 130 | cacheKey, 131 | JSON.stringify({ 132 | cache: hook.result, 133 | expiresOn: dayjs().add(duration, "s"), 134 | }) 135 | ); 136 | 137 | client.EXPIRE(cacheKey, duration); 138 | 139 | if (options.env !== "test" && ENABLE_REDIS_CACHE_LOGGER === "true") { 140 | console.log(`[redis] added ${cacheKey} to the cache.`); 141 | console.log( 142 | `> Expires in ${dayjs.duration(duration, "s").humanize()}.` 143 | ); 144 | } 145 | 146 | resolve(hook); 147 | }); 148 | } catch (err) { 149 | console.error(err); 150 | return Promise.resolve(hook); 151 | } 152 | }; 153 | }; 154 | 155 | async function purgeGroup(client, group, prefix = config.channel) { 156 | return new Promise((resolve, reject) => { 157 | let cursor = 0; 158 | const scan = async () => { 159 | const reply = await client 160 | .SCAN(cursor, { MATCH: `${prefix}${group}*`, COUNT: 1000 }) 161 | .catch((err) => reject(err)); 162 | 163 | cursor = reply.cursor; 164 | const keys = reply.keys; 165 | 166 | for (key of keys) { 167 | await client.del(key); 168 | } 169 | 170 | if (cursor !== 0) return scan(); 171 | 172 | resolve(); 173 | }; 174 | return scan(); 175 | }); 176 | } 177 | 178 | module.exports.purge = (passedOptions = {}) => { 179 | if (DISABLE_REDIS_CACHE) { 180 | return (hook) => hook; 181 | } 182 | 183 | return function (hook) { 184 | try { 185 | return new Promise((resolve) => { 186 | const client = hook.app.get("redisClient"); 187 | const options = { ...defaults, ...passedOptions }; 188 | const { prefix } = hook.app.get("redis"); 189 | const group = 190 | typeof options.cacheGroupKey === "function" 191 | ? hashCode(`group-${options.cacheGroupKey(hook)}`) 192 | : hashCode(`group-${hook.path || "general"}`); 193 | 194 | if (!client) { 195 | return { 196 | message: "Redis unavailable", 197 | status: HTTP_SERVER_ERROR, 198 | }; 199 | } 200 | 201 | purgeGroup(client, group, prefix).catch((err) => 202 | console.error({ 203 | message: err.message, 204 | status: HTTP_SERVER_ERROR, 205 | }) 206 | ); 207 | 208 | // do not wait for purge to resolve 209 | resolve(hook); 210 | }); 211 | } catch (err) { 212 | console.error(err); 213 | return Promise.resolve(hook); 214 | } 215 | }; 216 | }; 217 | 218 | module.exports.purgeVods = () => { 219 | return function (hook) { 220 | try { 221 | return new Promise((resolve) => { 222 | const client = hook.app.get("redisClient"); 223 | const { prefix } = hook.app.get("redis"); 224 | const group = hashCode(`group-vods`); 225 | 226 | if (!client) { 227 | return { 228 | message: "Redis unavailable", 229 | status: HTTP_SERVER_ERROR, 230 | }; 231 | } 232 | 233 | purgeGroup(client, group, prefix).catch((err) => 234 | console.error({ 235 | message: err.message, 236 | status: HTTP_SERVER_ERROR, 237 | }) 238 | ); 239 | 240 | // do not wait for purge to resolve 241 | resolve(hook); 242 | }); 243 | } catch (err) { 244 | console.error(err); 245 | return Promise.resolve(hook); 246 | } 247 | }; 248 | }; 249 | -------------------------------------------------------------------------------- /src/services/emotes/emotes.class.js: -------------------------------------------------------------------------------- 1 | const { SequelizeService } = require('feathers-sequelize'); 2 | 3 | exports.Emotes = class Emotes extends SequelizeService { 4 | 5 | }; 6 | -------------------------------------------------------------------------------- /src/services/emotes/emotes.hooks.js: -------------------------------------------------------------------------------- 1 | const { disallow, iff, isProvider } = require("feathers-hooks-common"); 2 | const redisCache = require("../cache"); 3 | 4 | module.exports = { 5 | before: { 6 | all: [], 7 | find: [iff(isProvider("external"), redisCache.before())], 8 | get: [iff(isProvider("external"), redisCache.before())], 9 | create: [disallow("external")], 10 | update: [disallow("external")], 11 | patch: [disallow("external")], 12 | remove: [disallow("external")], 13 | }, 14 | 15 | after: { 16 | all: [], 17 | find: [ 18 | iff(isProvider("external"), redisCache.after({ expiration: 3600 * 24 })), 19 | ], 20 | get: [ 21 | iff(isProvider("external"), redisCache.after({ expiration: 3600 * 24 })), 22 | ], 23 | create: [redisCache.purge()], 24 | update: [redisCache.purge()], 25 | patch: [redisCache.purge()], 26 | remove: [redisCache.purge()], 27 | }, 28 | 29 | error: { 30 | all: [], 31 | find: [], 32 | get: [], 33 | create: [], 34 | update: [], 35 | patch: [], 36 | remove: [], 37 | }, 38 | }; 39 | -------------------------------------------------------------------------------- /src/services/emotes/emotes.service.js: -------------------------------------------------------------------------------- 1 | // Initializes the `emotes` service on path `/emotes` 2 | const { Emotes } = require('./emotes.class'); 3 | const createModel = require('../../models/emotes.model'); 4 | const hooks = require('./emotes.hooks'); 5 | 6 | module.exports = function (app) { 7 | const options = { 8 | Model: createModel(app), 9 | paginate: app.get('paginate') 10 | }; 11 | 12 | // Initialize our service with any options it requires 13 | app.use('/emotes', new Emotes(options, app)); 14 | 15 | // Get our initialized service so that we can register hooks 16 | const service = app.service('emotes'); 17 | 18 | service.hooks(hooks); 19 | }; 20 | -------------------------------------------------------------------------------- /src/services/games/games.class.js: -------------------------------------------------------------------------------- 1 | const { SequelizeService } = require('feathers-sequelize'); 2 | 3 | exports.Games = class Games extends SequelizeService { 4 | 5 | }; 6 | -------------------------------------------------------------------------------- /src/services/games/games.hooks.js: -------------------------------------------------------------------------------- 1 | const { disallow, iff, isProvider } = require("feathers-hooks-common"); 2 | const redisCache = require("../cache"); 3 | const include = require("./include"); 4 | 5 | module.exports = { 6 | before: { 7 | all: [], 8 | find: [iff(isProvider("external"), redisCache.before(), include())], 9 | get: [iff(isProvider("external"), redisCache.before(), include())], 10 | create: [disallow("external")], 11 | update: [disallow("external")], 12 | patch: [disallow("external")], 13 | remove: [disallow("external")], 14 | }, 15 | 16 | after: { 17 | all: [], 18 | find: [ 19 | iff(isProvider("external"), redisCache.after({ expiration: 3600 * 24 })), 20 | ], 21 | get: [ 22 | iff(isProvider("external"), redisCache.after({ expiration: 3600 * 24 })), 23 | ], 24 | create: [redisCache.purge(), redisCache.purgeVods()], 25 | update: [redisCache.purge(), redisCache.purgeVods()], 26 | patch: [redisCache.purge(), redisCache.purgeVods()], 27 | remove: [redisCache.purge(), redisCache.purgeVods()], 28 | }, 29 | 30 | error: { 31 | all: [], 32 | find: [], 33 | get: [], 34 | create: [], 35 | update: [], 36 | patch: [], 37 | remove: [], 38 | }, 39 | }; 40 | -------------------------------------------------------------------------------- /src/services/games/games.service.js: -------------------------------------------------------------------------------- 1 | // Initializes the `games` service on path `/games` 2 | const { Games } = require('./games.class'); 3 | const createModel = require('../../models/games.model'); 4 | const hooks = require('./games.hooks'); 5 | 6 | module.exports = function (app) { 7 | const options = { 8 | Model: createModel(app), 9 | paginate: app.get('paginate') 10 | }; 11 | 12 | // Initialize our service with any options it requires 13 | app.use('/games', new Games(options, app)); 14 | 15 | // Get our initialized service so that we can register hooks 16 | const service = app.service('games'); 17 | 18 | service.hooks(hooks); 19 | }; 20 | -------------------------------------------------------------------------------- /src/services/games/include.js: -------------------------------------------------------------------------------- 1 | module.exports = () => { 2 | return async (context) => { 3 | const sequelize = context.app.get("sequelizeClient"); 4 | const { vods, games } = sequelize.models; 5 | context.params.sequelize = { 6 | include: [{ model: vods, include: [games] }], 7 | raw: false, 8 | }; 9 | return context; 10 | }; 11 | }; 12 | -------------------------------------------------------------------------------- /src/services/index.js: -------------------------------------------------------------------------------- 1 | const vods = require('./vods/vods.service.js'); 2 | const logs = require('./logs/logs.service.js'); 3 | const emotes = require('./emotes/emotes.service.js'); 4 | const games = require('./games/games.service.js'); 5 | const streams = require('./streams/streams.service.js'); 6 | // eslint-disable-next-line no-unused-vars 7 | module.exports = function (app) { 8 | app.configure(vods); 9 | app.configure(logs); 10 | app.configure(emotes); 11 | app.configure(games); 12 | app.configure(streams); 13 | }; 14 | -------------------------------------------------------------------------------- /src/services/logs/logs.class.js: -------------------------------------------------------------------------------- 1 | const { SequelizeService } = require('feathers-sequelize'); 2 | 3 | exports.Logs = class Logs extends SequelizeService { 4 | 5 | }; 6 | -------------------------------------------------------------------------------- /src/services/logs/logs.hooks.js: -------------------------------------------------------------------------------- 1 | const { disallow } = require("feathers-hooks-common"); 2 | const modify = require("./modify"); 3 | 4 | module.exports = { 5 | before: { 6 | all: [disallow("external")], 7 | find: [], 8 | get: [], 9 | create: [], 10 | update: [], 11 | patch: [], 12 | remove: [], 13 | }, 14 | 15 | after: { 16 | all: [], 17 | find: [modify()], 18 | get: [], 19 | create: [], 20 | update: [], 21 | patch: [], 22 | remove: [], 23 | }, 24 | 25 | error: { 26 | all: [], 27 | find: [], 28 | get: [], 29 | create: [], 30 | update: [], 31 | patch: [], 32 | remove: [], 33 | }, 34 | }; 35 | -------------------------------------------------------------------------------- /src/services/logs/logs.service.js: -------------------------------------------------------------------------------- 1 | // Initializes the `logs` service on path `/logs` 2 | const { Logs } = require("./logs.class"); 3 | const createModel = require("../../models/logs.model"); 4 | const hooks = require("./logs.hooks"); 5 | 6 | module.exports = function (app) { 7 | const options = { 8 | Model: createModel(app), 9 | paginate: { 10 | default: 10, 11 | max: 100, 12 | }, 13 | multi: true, 14 | }; 15 | 16 | // Initialize our service with any options it requires 17 | app.use("/logs", new Logs(options, app)); 18 | 19 | // Get our initialized service so that we can register hooks 20 | const service = app.service("logs"); 21 | 22 | service.hooks(hooks); 23 | }; 24 | -------------------------------------------------------------------------------- /src/services/logs/modify.js: -------------------------------------------------------------------------------- 1 | module.exports = function () { 2 | return (context) => { 3 | if(context.result.data) { 4 | for (let i = 0; i < context.result.data.length; i++) { 5 | context.result.data[i].content_offset_seconds = parseFloat( 6 | context.result.data[i].content_offset_seconds 7 | ); 8 | } 9 | } 10 | }; 11 | }; 12 | -------------------------------------------------------------------------------- /src/services/streams/streams.class.js: -------------------------------------------------------------------------------- 1 | const { SequelizeService } = require('feathers-sequelize'); 2 | 3 | exports.Streams = class Streams extends SequelizeService { 4 | 5 | }; 6 | -------------------------------------------------------------------------------- /src/services/streams/streams.hooks.js: -------------------------------------------------------------------------------- 1 | const { disallow } = require("feathers-hooks-common"); 2 | 3 | module.exports = { 4 | before: { 5 | all: [], 6 | find: [], 7 | get: [], 8 | create: [disallow("external")], 9 | update: [disallow("external")], 10 | patch: [disallow("external")], 11 | remove: [disallow("external")], 12 | }, 13 | 14 | after: { 15 | all: [], 16 | find: [], 17 | get: [], 18 | create: [], 19 | update: [], 20 | patch: [], 21 | remove: [], 22 | }, 23 | 24 | error: { 25 | all: [], 26 | find: [], 27 | get: [], 28 | create: [], 29 | update: [], 30 | patch: [], 31 | remove: [], 32 | }, 33 | }; 34 | -------------------------------------------------------------------------------- /src/services/streams/streams.service.js: -------------------------------------------------------------------------------- 1 | // Initializes the `streams` service on path `/streams` 2 | const { Streams } = require('./streams.class'); 3 | const createModel = require('../../models/streams.model'); 4 | const hooks = require('./streams.hooks'); 5 | 6 | module.exports = function (app) { 7 | const options = { 8 | Model: createModel(app), 9 | paginate: app.get('paginate') 10 | }; 11 | 12 | // Initialize our service with any options it requires 13 | app.use('/streams', new Streams(options, app)); 14 | 15 | // Get our initialized service so that we can register hooks 16 | const service = app.service('streams'); 17 | 18 | service.hooks(hooks); 19 | }; 20 | -------------------------------------------------------------------------------- /src/services/vods/include.js: -------------------------------------------------------------------------------- 1 | module.exports = () => { 2 | return async (context) => { 3 | const sequelize = context.app.get("sequelizeClient"); 4 | const { games } = sequelize.models; 5 | context.params.sequelize = { 6 | include: [{ model: games }], 7 | raw: false, 8 | }; 9 | return context; 10 | }; 11 | }; 12 | 13 | /** 14 | * Implment ilike search in chapters array. 15 | */ 16 | module.exports.games = () => { 17 | return async (context) => { 18 | const sequelize = context.app.get("sequelizeClient"); 19 | if (!context.params.query.chapters) return context; 20 | if (!context.params.query.chapters.name) return context; 21 | context.params.sequelize = { 22 | ...context.params.sequelize, 23 | where: sequelize.literal( 24 | `"vods"."chapters" @? '$[*] ? (@.name like_regex ".*${context.params.query.chapters.name}.*" flag "i")'` 25 | ), 26 | }; 27 | return context; 28 | }; 29 | }; 30 | -------------------------------------------------------------------------------- /src/services/vods/vods.class.js: -------------------------------------------------------------------------------- 1 | const { SequelizeService } = require('feathers-sequelize'); 2 | 3 | exports.Vods = class Vods extends SequelizeService { 4 | 5 | }; 6 | -------------------------------------------------------------------------------- /src/services/vods/vods.hooks.js: -------------------------------------------------------------------------------- 1 | const { disallow, iff, isProvider } = require("feathers-hooks-common"); 2 | const redisCache = require("../cache"); 3 | const include = require("./include"); 4 | 5 | module.exports = { 6 | before: { 7 | all: [], 8 | find: [ 9 | iff( 10 | isProvider("external"), 11 | redisCache.before(), 12 | include(), 13 | include.games() 14 | ), 15 | iff(isProvider("server"), include(), include.games()), 16 | ], 17 | get: [iff(isProvider("external"), redisCache.before(), include())], 18 | create: [disallow("external")], 19 | update: [disallow("external")], 20 | patch: [disallow("external")], 21 | remove: [disallow("external")], 22 | }, 23 | 24 | after: { 25 | all: [], 26 | find: [ 27 | iff(isProvider("external"), redisCache.after({ expiration: 3600 * 24 })), 28 | ], 29 | get: [ 30 | iff(isProvider("external"), redisCache.after({ expiration: 3600 * 24 })), 31 | ], 32 | create: [redisCache.purge()], 33 | update: [redisCache.purge()], 34 | patch: [redisCache.purge()], 35 | remove: [redisCache.purge()], 36 | }, 37 | 38 | error: { 39 | all: [], 40 | find: [], 41 | get: [], 42 | create: [], 43 | update: [], 44 | patch: [], 45 | remove: [], 46 | }, 47 | }; 48 | -------------------------------------------------------------------------------- /src/services/vods/vods.service.js: -------------------------------------------------------------------------------- 1 | // Initializes the `vods` service on path `/vods` 2 | const { Vods } = require("./vods.class"); 3 | const createModel = require("../../models/vods.model"); 4 | const hooks = require("./vods.hooks"); 5 | const { limiter } = require("../../middleware/rateLimit"); 6 | 7 | module.exports = function (app) { 8 | const options = { 9 | Model: createModel(app), 10 | paginate: app.get("paginate"), 11 | }; 12 | 13 | // Initialize our service with any options it requires 14 | app.use("/vods", limiter(app), new Vods(options, app)); 15 | 16 | // Get our initialized service so that we can register hooks 17 | const service = app.service("vods"); 18 | 19 | service.hooks(hooks); 20 | }; 21 | -------------------------------------------------------------------------------- /test/app.test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert').strict; 2 | const axios = require('axios'); 3 | const url = require('url'); 4 | const app = require('../src/app'); 5 | 6 | const port = app.get('port') || 8998; 7 | const getUrl = pathname => url.format({ 8 | hostname: app.get('host') || 'localhost', 9 | protocol: 'http', 10 | port, 11 | pathname 12 | }); 13 | 14 | describe('Feathers application tests', () => { 15 | let server; 16 | 17 | before(function(done) { 18 | server = app.listen(port); 19 | server.once('listening', () => done()); 20 | }); 21 | 22 | after(function(done) { 23 | server.close(done); 24 | }); 25 | 26 | it('starts and shows the index page', async () => { 27 | const { data } = await axios.get(getUrl()); 28 | 29 | assert.ok(data.indexOf('') !== -1); 30 | }); 31 | 32 | describe('404', function() { 33 | it('shows a 404 HTML page', async () => { 34 | try { 35 | await axios.get(getUrl('path/to/nowhere'), { 36 | headers: { 37 | 'Accept': 'text/html' 38 | } 39 | }); 40 | assert.fail('should never get here'); 41 | } catch (error) { 42 | const { response } = error; 43 | 44 | assert.equal(response.status, 404); 45 | assert.ok(response.data.indexOf('') !== -1); 46 | } 47 | }); 48 | 49 | it('shows a 404 JSON error without stack trace', async () => { 50 | try { 51 | await axios.get(getUrl('path/to/nowhere'), { 52 | json: true 53 | }); 54 | assert.fail('should never get here'); 55 | } catch (error) { 56 | const { response } = error; 57 | 58 | assert.equal(response.status, 404); 59 | assert.equal(response.data.code, 404); 60 | assert.equal(response.data.message, 'Page not found'); 61 | assert.equal(response.data.name, 'NotFound'); 62 | } 63 | }); 64 | }); 65 | }); 66 | -------------------------------------------------------------------------------- /test/services/emotes.test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const app = require('../../src/app'); 3 | 4 | describe('\'emotes\' service', () => { 5 | it('registered the service', () => { 6 | const service = app.service('emotes'); 7 | 8 | assert.ok(service, 'Registered the service'); 9 | }); 10 | }); 11 | -------------------------------------------------------------------------------- /test/services/games.test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const app = require('../../src/app'); 3 | 4 | describe('\'games\' service', () => { 5 | it('registered the service', () => { 6 | const service = app.service('games'); 7 | 8 | assert.ok(service, 'Registered the service'); 9 | }); 10 | }); 11 | -------------------------------------------------------------------------------- /test/services/logs.test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const app = require('../../src/app'); 3 | 4 | describe('\'logs\' service', () => { 5 | it('registered the service', () => { 6 | const service = app.service('logs'); 7 | 8 | assert.ok(service, 'Registered the service'); 9 | }); 10 | }); 11 | -------------------------------------------------------------------------------- /test/services/streams.test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const app = require('../../src/app'); 3 | 4 | describe('\'streams\' service', () => { 5 | it('registered the service', () => { 6 | const service = app.service('streams'); 7 | 8 | assert.ok(service, 'Registered the service'); 9 | }); 10 | }); 11 | -------------------------------------------------------------------------------- /test/services/vods.test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const app = require('../../src/app'); 3 | 4 | describe('\'vods\' service', () => { 5 | it('registered the service', () => { 6 | const service = app.service('vods'); 7 | 8 | assert.ok(service, 'Registered the service'); 9 | }); 10 | }); 11 | --------------------------------------------------------------------------------