├── .gitignore ├── _inputs └── .keep ├── _outputs └── .keep ├── app.js ├── index.m3u8 ├── keys └── .keep ├── mt-downloader-benchmark.js ├── package.json ├── presets ├── highlight_mp4.js ├── hls.js └── m3u8_to_mp4.js └── readme.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | *.pid.lock 11 | 12 | # Directory for instrumented libs generated by jscoverage/JSCover 13 | lib-cov 14 | 15 | # Coverage directory used by tools like istanbul 16 | coverage 17 | 18 | # nyc test coverage 19 | .nyc_output 20 | 21 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 22 | .grunt 23 | 24 | # node-waf configuration 25 | .lock-wscript 26 | 27 | # Compiled binary addons (http://nodejs.org/api/addons.html) 28 | build/Release 29 | 30 | # Dependency directories 31 | node_modules/ 32 | jspm_packages 33 | 34 | # Optional npm cache directory 35 | .npm 36 | 37 | # Optional eslint cache 38 | .eslintcache 39 | 40 | # Optional REPL history 41 | .node_repl_history 42 | 43 | # Output of 'npm pack' 44 | *.tgz 45 | 46 | # Yarn Integrity file 47 | .yarn-integrity 48 | 49 | AWS_access_keys 50 | broadcast-cx-bda0296621a4.json 51 | *.mp4 52 | !CX_brand_ending_stinger.mp4 53 | 54 | # Elastic Beanstalk Files 55 | .elasticbeanstalk/* 56 | !.elasticbeanstalk/*.cfg.yml 57 | !.elasticbeanstalk/*.global.yml 58 | 59 | _outputs/hls/*.ts 60 | _outputs/*.m3u8 61 | _outputs/*.m4a 62 | _outputs/hls/*.m3u8 63 | _outputs/*.mp4 64 | _outputs/thumbs/*.jpg 65 | keys/*.json 66 | -------------------------------------------------------------------------------- /_inputs/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyk2/cloud-transcoder/f1a52633d2ca6ce3fe98cc1d04ad07a8afc0e806/_inputs/.keep -------------------------------------------------------------------------------- /_outputs/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyk2/cloud-transcoder/f1a52633d2ca6ce3fe98cc1d04ad07a8afc0e806/_outputs/.keep -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs-extra'); 2 | var restify = require('restify'); 3 | var ffmpeg = require('fluent-ffmpeg'); 4 | const WebSocket = require('ws'); 5 | var dir = require('node-dir'); 6 | var path = require('path'); 7 | var request = require('request'); 8 | var async = require('async'); 9 | const uuidv4 = require('uuid/v4') 10 | var m3u8Parser = require('m3u8-parser'); 11 | var Raven = require('raven'); 12 | 13 | var MIME_TYPE_LUT = { 14 | '.m3u8': 'application/x-mpegURL', 15 | '.ts': 'video/MP2T', 16 | '.jpg': 'image/jpeg', 17 | '.mp4': 'video/mp4', 18 | '.m4a': 'audio/mp4' 19 | } 20 | 21 | if(process.env.NODE_ENV == 'production') { 22 | _INPUT_PATH = process.env.PWD + '/_inputs'; 23 | _OUTPUT_PATH = process.env.PWD + '/_outputs'; 24 | _PRESETS_PATH = process.env.PWD + '/presets'; 25 | _API_HOST = 'http://api.broadcast.cx'; 26 | _PORT = 8080; // 8080 forwarded to 80 with iptables rule 27 | _WS_PORT = 8081; 28 | Raven.config('https://c790451322a743ea89955afd471c2985:2b9c188e39444388a717d04b73b3ad5f@sentry.io/249073').install(); 29 | 30 | } else { // Running local on development 31 | _INPUT_PATH = '/Users/XYK/Desktop/Dropbox/broadcast.cx-ffmpeg-runner/_inputs'; 32 | _OUTPUT_PATH = '/Users/XYK/Desktop/Dropbox/broadcast.cx-ffmpeg-runner/_outputs'; 33 | _PRESETS_PATH = '/Users/XYK/Desktop/Dropbox/broadcast.cx-ffmpeg-runner/presets'; 34 | _API_HOST = 'http://local.broadcast.cx:8087'; 35 | ffmpeg.setFfmpegPath('/Users/XYK/Desktop/ffmpeg'); // Explicitly set ffmpeg and ffprobe paths 36 | ffmpeg.setFfprobePath('/Users/XYK/Desktop/ffprobe'); 37 | _PORT = 8082; 38 | _WS_PORT = 8081; 39 | Raven.config(null).install(); 40 | } 41 | 42 | // Grab machine details from GCP if available, otherwise null 43 | var machine_details = {}; 44 | 45 | request({ 46 | url: 'http://metadata.google.internal/computeMetadata/v1/instance/?recursive=true', 47 | headers: { 'Metadata-Flavor': 'Google' }, 48 | timeout: 1500 49 | }, function(error, response, body) { 50 | if(error) return; // If times out or no response at all 51 | else machine_details = JSON.parse(body); 52 | }); 53 | 54 | var _transcodeInProgress = false; 55 | 56 | var google_cloud = require('google-cloud')({ 57 | projectId: 'broadcast-cx', 58 | keyFilename: 'keys/broadcast-cx-bda0296621a4.json' 59 | }); 60 | 61 | var gcs = google_cloud.storage(); 62 | var bucket = gcs.bucket('broadcast-cx-raw-recordings'); 63 | var dest_bucket = gcs.bucket('cx-video-content'); 64 | 65 | var server = restify.createServer({ 66 | name: 'ffmpeg-runner' 67 | }); 68 | 69 | server.pre(restify.pre.sanitizePath()); 70 | server.use(restify.acceptParser(server.acceptable)); 71 | server.use(restify.queryParser({ mapParams: false })); // Parses URL queries, i.e. ?name=hello&gender=male 72 | server.use(restify.bodyParser()); 73 | server.use(restify.gzipResponse()); // Gzip by default if accept-encoding: gzip is set on request 74 | server.pre(restify.CORS()); // Enable CORS headers 75 | server.use(restify.fullResponse()); 76 | 77 | server.get('/healthcheck', function(req, res) { 78 | res.send(machine_details); 79 | }); 80 | 81 | const wss = new WebSocket.Server({ port: _WS_PORT }); 82 | 83 | wss.broadcast = function broadcast(data) { 84 | console.log(data); 85 | wss.clients.forEach(function each(client) { 86 | if(client.readyState === WebSocket.OPEN) { 87 | client.send(data); 88 | } 89 | }); 90 | }; 91 | 92 | wss.on('connection', function connection(ws) { 93 | ws.send(JSON.stringify({'event': 'success', 'message': 'Connected to ffmpeg-runner.'})); 94 | }); 95 | 96 | 97 | HD_720P_TRANSCODE = function(filename, prefix, startTime, endTime, callback) { 98 | _HD_720P = ffmpeg(filename, { presets: _PRESETS_PATH }).preset('hls') 99 | .videoBitrate(3000) 100 | .audioBitrate('128k') 101 | .renice(-10); 102 | 103 | if(startTime && endTime) { 104 | _HD_720P.seekInput(startTime-10 < 0 ? 0 : startTime - 10); 105 | _HD_720P.outputOptions(['-ss ' + (startTime).toFixed(2), '-t ' + (endTime - startTime), '-copyts']); 106 | } 107 | 108 | if(path.extname(filename) == '.avi') { 109 | _HD_720P.outputOptions('-filter:v', "yadif=0, scale=w=1280:h=720'"); 110 | _HD_720P.outputOptions('-pix_fmt', 'yuv420p'); 111 | } 112 | else _HD_720P.outputOptions('-filter:v', 'scale=w=1280:h=720'); 113 | 114 | _HD_720P.on('start', function(commandLine) { 115 | wss.broadcast(JSON.stringify({'event': 'm3u8', 'status': 'start', 'rendition': '720P_3000K', 'command': commandLine})); 116 | 117 | }) 118 | .on('progress', function(progress) { 119 | progress['event'] = 'progress'; 120 | progress['rendition'] = '720P_3000K'; 121 | wss.broadcast(JSON.stringify(progress)); 122 | }) 123 | .on('stderr', function(stderrLine) { 124 | //console.log('Stderr output: ' + stderrLine); 125 | }) 126 | .on('error', function(err, stdout, stderr) { 127 | wss.broadcast(JSON.stringify({'event': 'error', 'message': err.message})); 128 | return callback(err.message); 129 | }) 130 | .on('end', function(stdout, stderr) { 131 | wss.broadcast(JSON.stringify({'event': 'm3u8', 'status': 'complete', 'rendition': '720P_3000K'})); 132 | 133 | ffmpeg(_OUTPUT_PATH + '/hls/720p_3000k.m3u8', { presets: _PRESETS_PATH }) // Concatenate M3U8 playlist into MP4 with moovatom at front 134 | .preset('m3u8_to_mp4') 135 | .on('start', function(commandLine) { 136 | wss.broadcast(JSON.stringify({'event': 'mp4', 'status': 'start', 'rendition': '720P_3000K', 'command': commandLine})); 137 | }) 138 | .on('end', function(stdout, stderr) { 139 | wss.broadcast(JSON.stringify({'event': 'mp4', 'status': 'complete', 'rendition': '720P_3000K'})); 140 | 141 | return callback(null, prefix + '_720p_3000k.mp4'); 142 | }) 143 | .saveToFile(_OUTPUT_PATH + '/' + prefix + '_720p_3000k.mp4') 144 | 145 | }) 146 | .saveToFile(_OUTPUT_PATH + '/hls/720p_3000k.m3u8'); 147 | } 148 | 149 | SD_480P_TRANSCODE = function(filename, prefix, startTime, endTime, callback) { 150 | _SD_480P = ffmpeg(filename, { presets: _PRESETS_PATH }).preset('hls') 151 | .videoBitrate(1500) 152 | .audioBitrate('128k') 153 | .renice(-10); 154 | 155 | if(startTime && endTime) { 156 | _SD_480P.seekInput(startTime-10 < 0 ? 0 : startTime - 10); 157 | _SD_480P.outputOptions(['-ss ' + (startTime).toFixed(2), '-t ' + (endTime - startTime), '-copyts']); 158 | } 159 | 160 | if(path.extname(filename) == '.avi') { 161 | _SD_480P.outputOptions('-filter:v', "yadif=0, scale=w=854:h=480'"); 162 | _SD_480P.outputOptions('-pix_fmt', 'yuv420p'); 163 | } 164 | else _SD_480P.outputOptions('-filter:v', 'scale=w=854:h=480'); 165 | 166 | 167 | _SD_480P.on('start', function(commandLine) { 168 | wss.broadcast(JSON.stringify({'event': 'm3u8', 'status': 'start', 'rendition': '480P_1500K', 'command': commandLine})); 169 | }) 170 | .on('progress', function(progress) { 171 | progress['event'] = 'progress'; 172 | progress['rendition'] = '480P_1500K'; 173 | wss.broadcast(JSON.stringify(progress)); 174 | }) 175 | .on('stderr', function(stderrLine) { 176 | //console.log('Stderr output: ' + stderrLine); 177 | }) 178 | .on('error', function(err, stdout, stderr) { 179 | wss.broadcast(JSON.stringify({'event': 'error', 'message': err.message})); 180 | return callback(err.message); 181 | }) 182 | .on('end', function(stdout, stderr) { 183 | wss.broadcast(JSON.stringify({'event': 'm3u8', 'status': 'complete', 'rendition': '480P_1500K'})); 184 | 185 | ffmpeg(_OUTPUT_PATH + '/hls/480p_1500k.m3u8', { presets: _PRESETS_PATH }) // Concatenate M3U8 playlist into MP4 with moovatom at front 186 | .preset('m3u8_to_mp4') 187 | .on('start', function(commandLine) { 188 | wss.broadcast(JSON.stringify({'event': 'mp4', 'status': 'start', 'rendition': '480P_1500K', 'command': commandLine})); 189 | }) 190 | .on('end', function(stdout, stderr) { 191 | wss.broadcast(JSON.stringify({'event': 'mp4', 'status': 'complete', 'rendition': '480P_1500K'})); 192 | 193 | return callback(null, prefix + '_480p_1500k.mp4'); 194 | }) 195 | .saveToFile(_OUTPUT_PATH + '/' + prefix + '_480p_1500k.mp4') 196 | 197 | }) 198 | .saveToFile(_OUTPUT_PATH + '/hls/480p_1500k.m3u8'); 199 | } 200 | 201 | SD_360P_TRANSCODE = function(filename, prefix, startTime, endTime, callback) { 202 | _SD_360P = ffmpeg(filename, { presets: _PRESETS_PATH }).preset('hls') 203 | .videoBitrate(850) 204 | .audioBitrate('128k') 205 | .renice(-10); 206 | 207 | if(startTime && endTime) { 208 | _SD_360P.seekInput(startTime-10 < 0 ? 0 : startTime - 10); 209 | _SD_360P.outputOptions(['-ss ' + (startTime).toFixed(2), '-t ' + (endTime - startTime), '-copyts']); 210 | } 211 | 212 | if(path.extname(filename) == '.avi') { 213 | _SD_360P.outputOptions('-filter:v', "yadif=0, scale=w=640:h=360'"); 214 | _SD_360P.outputOptions('-pix_fmt', 'yuv420p'); 215 | } 216 | else _SD_360P.outputOptions('-filter:v', 'scale=w=640:h=360'); 217 | 218 | 219 | _SD_360P.on('start', function(commandLine) { 220 | wss.broadcast(JSON.stringify({'event': 'm3u8', 'status': 'start', 'rendition': '360P_850K', 'command': commandLine})); 221 | }) 222 | .on('progress', function(progress) { 223 | progress['event'] = 'progress'; 224 | progress['rendition'] = '360P_850K'; 225 | wss.broadcast(JSON.stringify(progress)); 226 | }) 227 | .on('stderr', function(stderrLine) { 228 | //console.log('Stderr output: ' + stderrLine); 229 | }) 230 | .on('error', function(err, stdout, stderr) { 231 | wss.broadcast(JSON.stringify({'event': 'error', 'message': err.message})); 232 | return callback(err.message); 233 | }) 234 | .on('end', function(stdout, stderr) { 235 | wss.broadcast(JSON.stringify({'event': 'm3u8', 'status': 'complete', 'rendition': '360P_850K'})); 236 | 237 | ffmpeg(_OUTPUT_PATH + '/hls/360p_850k.m3u8', { presets: _PRESETS_PATH }) // Concatenate M3U8 playlist into MP4 with moovatom at front 238 | .preset('m3u8_to_mp4') 239 | .on('start', function(commandLine) { 240 | wss.broadcast(JSON.stringify({'event': 'mp4', 'status': 'start', 'rendition': '360P_850K', 'command': commandLine})); 241 | }) 242 | .on('end', function(stdout, stderr) { 243 | wss.broadcast(JSON.stringify({'event': 'mp4', 'status': 'complete', 'rendition': '360P_850K'})); 244 | 245 | return callback(null, prefix + '_360p_850k.mp4'); 246 | }) 247 | .saveToFile(_OUTPUT_PATH + '/' + prefix + '_360p_850k.mp4') 248 | 249 | }) 250 | .saveToFile(_OUTPUT_PATH + '/hls/360p_850k.m3u8'); 251 | } 252 | 253 | SD_240P_TRANSCODE = function(filename, prefix, startTime, endTime, callback) { 254 | _SD_240P = ffmpeg(filename, { presets: _PRESETS_PATH }).preset('hls') 255 | .videoBitrate(400) 256 | .audioBitrate('128k') 257 | .renice(-10); 258 | 259 | if(startTime && endTime) { 260 | _SD_240P.seekInput(startTime-10 < 0 ? 0 : startTime - 10); 261 | _SD_240P.outputOptions(['-ss ' + (startTime).toFixed(2), '-t ' + (endTime - startTime), '-copyts']); 262 | } 263 | 264 | if(path.extname(filename) == '.avi') { 265 | _SD_240P.outputOptions('-filter:v', "yadif=0, scale=w=352:h=240'"); 266 | _SD_240P.outputOptions('-pix_fmt', 'yuv420p'); 267 | } 268 | else _SD_240P.outputOptions('-filter:v', 'scale=w=352:h=240'); 269 | 270 | 271 | _SD_240P.on('start', function(commandLine) { 272 | wss.broadcast(JSON.stringify({'event': 'm3u8', 'status': 'start', 'rendition': '240P_400K', 'command': commandLine})); 273 | }) 274 | .on('progress', function(progress) { 275 | progress['event'] = 'progress'; 276 | progress['rendition'] = '240P_400K'; 277 | wss.broadcast(JSON.stringify(progress)); 278 | }) 279 | .on('stderr', function(stderrLine) { 280 | //console.log('Stderr output: ' + stderrLine); 281 | }) 282 | .on('error', function(err, stdout, stderr) { 283 | wss.broadcast(JSON.stringify({'event': 'error', 'message': err.message})); 284 | return callback(err.message); 285 | }) 286 | .on('end', function(stdout, stderr) { 287 | wss.broadcast(JSON.stringify({'event': 'm3u8', 'status': 'complete', 'rendition': '240P_400K'})); 288 | 289 | ffmpeg(_OUTPUT_PATH + '/hls/240p_400k.m3u8', { presets: _PRESETS_PATH }) // Concatenate M3U8 playlist into MP4 with moovatom at front 290 | .preset('m3u8_to_mp4') 291 | .on('start', function(commandLine) { 292 | wss.broadcast(JSON.stringify({'event': 'mp4', 'status': 'start', 'rendition': '240P_400K', 'command': commandLine})); 293 | }) 294 | .on('error', function(err, stdout, stderr) { 295 | console.log(err.message); 296 | }) 297 | .on('end', function(stdout, stderr) { 298 | wss.broadcast(JSON.stringify({'event': 'mp4', 'status': 'complete', 'rendition': '240P_400K'})); 299 | 300 | return callback(null, prefix + '_240p_400k.mp4'); 301 | }) 302 | .saveToFile(_OUTPUT_PATH + '/' + prefix + '_240p_400k.mp4') 303 | }) 304 | .saveToFile(_OUTPUT_PATH + '/hls/240p_400k.m3u8'); 305 | } 306 | 307 | AAC_128KBPS_HLS = function(filename, startTime, endTime, callback) { 308 | _AAC_128KBPS = ffmpeg(filename, { presets: _PRESETS_PATH }) 309 | .noVideo() 310 | .audioBitrate('128k') 311 | .outputOptions('-c:a', 'aac') 312 | .outputOptions('-hls_time', '6') 313 | .outputOptions('-hls_list_size', '0') 314 | .outputOptions('-f', 'hls'); 315 | 316 | if(startTime && endTime) { 317 | _AAC_128KBPS.seekInput(startTime-10 < 0 ? 0 : startTime - 10); 318 | _AAC_128KBPS.outputOptions(['-ss ' + (startTime).toFixed(2), '-t ' + (endTime - startTime), '-copyts']); 319 | } 320 | 321 | _AAC_128KBPS.on('start', function(commandLine) { 322 | wss.broadcast(JSON.stringify({'event': 'm3u8', 'status': 'start', 'rendition': 'AAC_128KBPS', 'command': commandLine})); 323 | }) 324 | .on('progress', function(progress) { 325 | progress['event'] = 'progress'; 326 | progress['rendition'] = 'AAC_128KBPS'; 327 | wss.broadcast(JSON.stringify(progress)); 328 | }) 329 | .on('stderr', function(stderrLine) { 330 | //console.log('Stderr output: ' + stderrLine); 331 | }) 332 | .on('error', function(err, stdout, stderr) { 333 | wss.broadcast(JSON.stringify({'event': 'error', 'message': err.message})); 334 | return callback(err.message); 335 | }) 336 | .on('end', function(stdout, stderr) { 337 | wss.broadcast(JSON.stringify({'event': 'M3U8', 'status': 'complete', 'rendition': 'AAC_128KBPS'})); 338 | return callback(null, '/hls/128kbps_aac.m3u8'); 339 | }) 340 | .saveToFile(_OUTPUT_PATH + '/hls/128kbps_aac.m3u8'); 341 | } 342 | 343 | AAC_128KBPS_M4A = function(filename, prefix, startTime, endTime, callback) { 344 | _AAC_128KBPS = ffmpeg(filename, { presets: _PRESETS_PATH }) 345 | .noVideo() 346 | .audioBitrate('128k') 347 | .outputOptions('-c:a', 'aac'); 348 | 349 | if(startTime && endTime) { 350 | _AAC_128KBPS.seekInput(startTime-10 < 0 ? 0 : startTime - 10); 351 | _AAC_128KBPS.outputOptions(['-ss ' + (startTime).toFixed(2), '-t ' + (endTime - startTime), '-copyts']); 352 | } 353 | 354 | _AAC_128KBPS.on('start', function(commandLine) { 355 | wss.broadcast(JSON.stringify({'event': 'aac', 'status': 'start', 'rendition': 'AAC_128KBPS', 'command': commandLine})); 356 | }) 357 | .on('progress', function(progress) { 358 | progress['event'] = 'progress'; 359 | progress['rendition'] = 'AAC_128KBPS'; 360 | wss.broadcast(JSON.stringify(progress)); 361 | }) 362 | .on('stderr', function(stderrLine) { 363 | //console.log('Stderr output: ' + stderrLine); 364 | }) 365 | .on('error', function(err, stdout, stderr) { 366 | wss.broadcast(JSON.stringify({'event': 'error', 'message': err.message})); 367 | return callback(err.message); 368 | }) 369 | .on('end', function(stdout, stderr) { 370 | wss.broadcast(JSON.stringify({'event': 'aac', 'status': 'complete', 'rendition': 'AAC_128KBPS'})); 371 | return callback(null, prefix + '_128kbps_aac.m4a'); 372 | }) 373 | .saveToFile(_OUTPUT_PATH + '/' + prefix + '_128kbps_aac.m4a'); 374 | } 375 | 376 | CREATE_THUMBNAILS = function(filename, startTime, endTime, callback) { 377 | _thumbnailFiles = []; 378 | 379 | _THUMBNAILS = ffmpeg(filename); 380 | _OPTIONS = { 381 | count: 5, // Will take screens at 20%, 40%, 60% and 80% of the video 382 | folder: _OUTPUT_PATH + '/thumbs', 383 | filename: 'thumbnail_%s_%00i_%r.jpg', 384 | size: '640x360' 385 | }; 386 | 387 | if(startTime && endTime) { // if start/end time specified, take screenshots in between startTime and endTime 388 | _OPTIONS.timemarks = []; 389 | _OPTIONS.timemarks.push(startTime); 390 | _OPTIONS.timemarks.push(startTime + ((endTime - startTime) / 3) * 1); 391 | _OPTIONS.timemarks.push(startTime + ((endTime - startTime) / 3) * 2); 392 | _OPTIONS.timemarks.push(startTime + ((endTime - startTime) / 3) * 3); 393 | _OPTIONS.timemarks.push(endTime); 394 | } 395 | 396 | _THUMBNAILS.on('filenames', function(filenames) { 397 | wss.broadcast(JSON.stringify({'event': 'thumbnail', 'status': 'start', 'files': filenames})); 398 | _thumbnailFiles = filenames; 399 | }) 400 | .on('end', function() { 401 | wss.broadcast(JSON.stringify({'event': 'thumbnail', 'status': 'complete'})); 402 | return callback(null, _thumbnailFiles); 403 | }) 404 | .on('error', function(err, stdout, stderr) { 405 | wss.broadcast(JSON.stringify({'event': 'error', 'message': err.message})); 406 | return callback(err.message); 407 | }) 408 | .screenshots(_OPTIONS); 409 | } 410 | 411 | CREATE_INDEX_M3U8 = function(callback) { 412 | fs.createReadStream('index.m3u8').pipe(fs.createWriteStream(_OUTPUT_PATH + '/hls/index.m3u8')); 413 | return callback(null, "/hls/index.m3u8"); 414 | } 415 | 416 | // Recursively retry upload to GCS if fails 417 | GCS_UPLOAD_RECURSIVE = function(file, options, callback) { 418 | dest_bucket.upload(file, options, function(err, gFileObj) { 419 | if(err) { 420 | console.log("File upload failed for " + file + ", trying again."); 421 | Raven.captureException(err); 422 | GCS_UPLOAD_RECURSIVE(file, options, callback); // retry if error 423 | return; 424 | } 425 | 426 | _ret = {'event': 'gcsupload', 'file': gFileObj.name}; 427 | wss.broadcast(JSON.stringify(_ret)); 428 | return callback(); 429 | }); 430 | } 431 | 432 | TRANSCODE_FILE_TO_HLS_AND_UPLOAD = function(filename, prefix, startTime, endTime, destination, _callback) { 433 | async.waterfall([ 434 | function(callback) { // Clear output directories 435 | fs.emptyDir(_OUTPUT_PATH, err => { // Clear out output path of old m3u8 files 436 | if (err) return _callback(err); 437 | fs.mkdirSync(path.join(_OUTPUT_PATH, 'hls')); 438 | fs.mkdirSync(path.join(_OUTPUT_PATH, 'thumbs')); 439 | callback(); 440 | }); 441 | }, 442 | function(callback) { // Run transcodes in parallel 443 | async.parallel([ // ORDER MATTERS HERE 444 | function(callback) { CREATE_THUMBNAILS(filename, startTime, endTime, callback); }, 445 | function(callback) { HD_720P_TRANSCODE(filename, prefix, startTime, endTime, callback); }, 446 | function(callback) { SD_480P_TRANSCODE(filename, prefix, startTime, endTime, callback); }, 447 | function(callback) { SD_360P_TRANSCODE(filename, prefix, startTime, endTime, callback); }, 448 | function(callback) { SD_240P_TRANSCODE(filename, prefix, startTime, endTime, callback); }, 449 | function(callback) { AAC_128KBPS_M4A(filename, prefix, startTime, endTime, callback); }, 450 | function(callback) { CREATE_INDEX_M3U8(callback); }, 451 | function(callback) { AAC_128KBPS_HLS(filename, startTime, endTime, callback); } 452 | ], 453 | function(err, results) { 454 | if(err) return _callback(err); 455 | 456 | // Get total duration of video from parsing output m3u8 457 | var parser = new m3u8Parser.Parser(); 458 | parser.push(fs.readFileSync(path.join(_OUTPUT_PATH, results[6].replace('index', '720p_3000k')), 'utf-8')); 459 | parser.end(); 460 | var parsed_mfst = parser.manifest; 461 | 462 | _total_duration = 0; 463 | for (segment in parsed_mfst.segments) _total_duration += parsed_mfst.segments[segment].duration; 464 | 465 | _ret = { 466 | filenames: results, 467 | duration: parseInt(_total_duration * 1000) 468 | }; 469 | 470 | console.log(_ret); 471 | callback(null, _ret); 472 | }); 473 | }, 474 | function(filenames, callback) { // Upload to GCS 475 | dir.files(_OUTPUT_PATH, function(err, files) { 476 | if (err) throw err; 477 | console.log('Files to upload: ' + files.length); 478 | 479 | async.eachOfLimit(files, 100, function(absolute_fn, key, callback) { 480 | if(!MIME_TYPE_LUT[path.extname(absolute_fn)]) return; // Only upload allowed extensions 481 | 482 | var _options = { // GCS options 483 | resumable: false, // default is true for files >5MB 484 | validation: false, // Disable crc32/md5 checksum validation 485 | destination: path.join(destination, path.relative(_OUTPUT_PATH, absolute_fn)), 486 | metadata: { 487 | contentType: MIME_TYPE_LUT[path.extname(absolute_fn)], 488 | cacheControl: 'public, max-age=31556926' 489 | } 490 | }; 491 | 492 | GCS_UPLOAD_RECURSIVE(absolute_fn, _options, callback); 493 | }, 494 | function(err) { 495 | console.log('Completed upload ' + files.length); 496 | _callback(null, filenames); 497 | }); 498 | }); 499 | } 500 | ]); 501 | } 502 | 503 | MASTER_GAME_FOOTAGE_HLS = function(filename, job, callback) { 504 | wss.broadcast(JSON.stringify({'event': 'mp4', 'status': 'pending', 'rendition': '240P_400K'})); 505 | wss.broadcast(JSON.stringify({'event': 'mp4', 'status': 'pending', 'rendition': '360P_850K'})); 506 | wss.broadcast(JSON.stringify({'event': 'mp4', 'status': 'pending', 'rendition': '480P_1500K'})); 507 | wss.broadcast(JSON.stringify({'event': 'mp4', 'status': 'pending', 'rendition': '720P_3000K'})); 508 | wss.broadcast(JSON.stringify({'event': 'download', 'status': 'start', 'file': filename})); 509 | _transcodeInProgress = true; 510 | 511 | uuid = uuidv4(); 512 | _GCS_BASEPATH = path.join('game_footage', uuid); 513 | 514 | async.waterfall([ 515 | function(callback) { // Clear input directory if running on prod 516 | if(process.env.NODE_ENV != 'production') return callback(); 517 | fs.emptyDir(_INPUT_PATH, err => { // Clear out input path so storage doesn't fill up on instance 518 | if (err) return callback(err); 519 | return callback(); 520 | }); 521 | }, 522 | function(callback) { // Download file 523 | if(fs.existsSync(path.join(_INPUT_PATH, filename))) { // If file already downloaded in local directory, use that instead of downloading again 524 | wss.broadcast(JSON.stringify({'event': 'download', 'status': 'complete', 'file': filename})); 525 | return callback(); 526 | 527 | } else { // TODO: upgrade to aria2c, significantly faster 528 | var _download_start_time = Date.now(); 529 | 530 | bucket.file(filename).download({ destination: path.join(_INPUT_PATH, filename) }, function(err) { 531 | if(err) { // Error handling (bucket file not found in GCS) 532 | wss.broadcast(JSON.stringify({'event': 'download', 'status': 'error', 'message': err.code + ' ' + err.message})); 533 | return callback(err.code + ' ' + err.message); 534 | } 535 | 536 | request.put({ url: `${_API_HOST}/v2/transcode/jobs/${job.id}/download_meta`, json: { download_time: parseInt(Date.now() - _download_start_time), download_method: 'google-cloud' }}, function(error, response, body) { }); 537 | 538 | wss.broadcast(JSON.stringify({'event': 'download', 'status': 'complete', 'file': filename})); 539 | return callback(); 540 | }); 541 | } 542 | }, 543 | function(callback) { 544 | TRANSCODE_FILE_TO_HLS_AND_UPLOAD(path.join(_INPUT_PATH, filename), uuid, null, null, _GCS_BASEPATH, callback); 545 | } 546 | ], function(err, results) { 547 | if(err) { 548 | Raven.captureException(err); 549 | return request.put({ 550 | url: _API_HOST + '/v2/transcode/jobs/' + job.id + '/error', 551 | method: 'PUT', 552 | json: { message: err } 553 | }, function(error, response, body) { 554 | _transcodeInProgress = false; 555 | console.log('PUTTING TO assets_transcode_queue: ERROR') 556 | }); 557 | } 558 | 559 | async.parallel([ 560 | function(callback) { // Update queue status to FINISHED 561 | console.log('PUTTING TO assets_transcode_queue: FINISHED'); 562 | request.put(_API_HOST + '/v2/transcode/jobs/' + job.id + '/finished', function(error, response, body) { callback(); }); 563 | }, 564 | function(callback) { // Update assets_game_footage table with urls 565 | console.log("PUTTING TO assets_game_footage " + '/v2/assets/game_footage/' + job.asset_game_footage_id); 566 | _PUT_BODY = { 567 | hls_playlist_url: 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, results.filenames[6]), 568 | mp4_240p: 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, results.filenames[4]), 569 | mp4_360p: 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, results.filenames[3]), 570 | mp4_480p: 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, results.filenames[2]), 571 | mp4_720p: 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, results.filenames[1]), 572 | thumbnails: JSON.stringify(results.filenames[0].map(function(e) {return 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, 'thumbs', e)})), 573 | duration: results.duration 574 | }; 575 | 576 | request.put({ 577 | url: _API_HOST + '/v2/assets/game_footage/' + job.asset_game_footage_id, 578 | method: 'PUT', 579 | json: _PUT_BODY 580 | }, function(error, response, body) { 581 | callback(); 582 | }); 583 | } 584 | ], function(err, response) { 585 | _transcodeInProgress = false; 586 | }); 587 | }); 588 | 589 | } 590 | 591 | MASTER_TRIM = function(filename, job, callback) { 592 | _transcodeInProgress = true; 593 | uuid = uuidv4(); 594 | 595 | _GCS_BASEPATH = path.join('events', uuid); 596 | 597 | async.waterfall([ 598 | function(callback) { 599 | if(!job.parameters || !('startTime' in job.parameters) || !('endTime' in job.parameters) || !job.parameters.api) { 600 | return callback("Incorrect / missing information in parameter."); 601 | } else { 602 | callback(); 603 | } 604 | }, 605 | function(callback) { // Clear input directory if running on prod 606 | if(process.env.NODE_ENV != 'production') return callback(); 607 | fs.emptyDir(_INPUT_PATH, err => { // Clear out input path so storage doesn't fill up on instance 608 | if (err) return callback(err); 609 | return callback(); 610 | }); 611 | }, 612 | function(callback) { // Download file 613 | if(fs.existsSync(path.join(_INPUT_PATH, filename))) { // If file already downloaded in local directory, use that instead of downloading again 614 | wss.broadcast(JSON.stringify({'event': 'download', 'status': 'complete', 'file': filename})); 615 | return callback(); 616 | 617 | } else { // TODO: upgrade to aria2c, significantly faster 618 | var _download_start_time = Date.now(); 619 | 620 | bucket.file(filename).download({ destination: path.join(_INPUT_PATH, filename) }, function(err) { 621 | if(err) { // Error handling (bucket file not found in GCS) 622 | wss.broadcast(JSON.stringify({'event': 'download', 'status': 'error', 'message': err.code + ' ' + err.message})); 623 | return callback(err.code + ' ' + err.message); 624 | } 625 | 626 | request.put({ url: `${_API_HOST}/v2/transcode/jobs/${job.id}/download_meta`, json: { download_time: parseInt(Date.now() - _download_start_time), download_method: 'google-cloud' }}, function(error, response, body) { }); 627 | 628 | wss.broadcast(JSON.stringify({'event': 'download', 'status': 'complete', 'file': filename})); 629 | return callback(); 630 | }); 631 | } 632 | }, 633 | function(callback) { 634 | TRANSCODE_FILE_TO_HLS_AND_UPLOAD(path.join(_INPUT_PATH, filename), uuid, job.parameters.startTime, job.parameters.endTime, _GCS_BASEPATH, callback); 635 | } 636 | ], function(err, results) { 637 | if(err) { 638 | Raven.captureException(err); 639 | return request.put({ 640 | url: _API_HOST + '/v2/transcode/jobs/' + job.id + '/error', 641 | method: 'PUT', 642 | json: { message: err } 643 | }, function(error, response, body) { 644 | _transcodeInProgress = false; 645 | console.log('PUTTING TO assets_transcode_queue: ERROR') 646 | }); 647 | } 648 | 649 | async.parallel([ 650 | function(callback) { // Update queue status to FINISHED 651 | console.log('PUTTING TO assets_transcode_queue: FINISHED'); 652 | request.put(_API_HOST + '/v2/transcode/jobs/' + job.id + '/finished', function(error, response, body) { callback(); }); 653 | }, 654 | function(callback) { // Update v2_videos table with urls 655 | console.log("POST TO v2_videos"); 656 | _POST_BODY = { 657 | hls_playlist_url: 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, results.filenames[6]), 658 | mp4_240p: 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, results.filenames[4]), 659 | mp4_360p: 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, results.filenames[3]), 660 | mp4_480p: 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, results.filenames[2]), 661 | mp4_720p: 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, results.filenames[1]), 662 | thumbnails: JSON.stringify(results.filenames[0].map(function(e) {return 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, 'thumbs', e)})), 663 | duration: results.duration, 664 | source_game_footage_id: job.asset_game_footage_id, 665 | uuid: uuid 666 | }; 667 | 668 | _POST_BODY = Object.assign(_POST_BODY, job.parameters.api); 669 | console.log(_POST_BODY); 670 | 671 | request.post({ 672 | url: _API_HOST + '/v2/videos', 673 | method: 'POST', 674 | json: _POST_BODY 675 | }, function(error, response, body) { 676 | callback(); 677 | }); 678 | } 679 | ], function(err, response) { 680 | _transcodeInProgress = false; 681 | }); 682 | }); 683 | } 684 | 685 | 686 | /* 687 | { 688 | source_asset_id: 1212, 689 | startTime: 10.0, 690 | endTime: 19.1, 691 | user_id: 1868, 692 | title: 'LOL', 693 | description: 'suckonthese' 694 | } 695 | */ 696 | 697 | MASTER_INDIVIDUAL_HIGHLIGHT = function(filename, job, callback) { 698 | wss.broadcast(JSON.stringify({'event': 'mp4', 'status': 'pending', 'rendition': '720P_2000K'})); 699 | wss.broadcast(JSON.stringify({'event': 'download', 'status': 'start', 'file': filename})); 700 | 701 | _transcodeInProgress = true; 702 | 703 | uuid = uuidv4(); 704 | 705 | _GCS_BASEPATH = path.join('individual_highlight', uuid); 706 | 707 | async.waterfall([ 708 | // Check for correct parameters in the job 709 | function(callback) { 710 | if(!job.parameters || !('startTime' in job.parameters) || !('endTime' in job.parameters)) { 711 | return callback("Incorrect / missing information in parameter."); 712 | } else { 713 | callback(); 714 | } 715 | }, 716 | // Clear input directory if running on prod 717 | function(callback) { 718 | if(process.env.NODE_ENV != 'production') return callback(); 719 | fs.emptyDir(_INPUT_PATH, err => { // Clear out input path so storage doesn't fill up on instance 720 | if (err) return callback(err); 721 | return callback(); 722 | }); 723 | }, 724 | // Download file if it doesn't exist 725 | function(callback) { 726 | if(fs.existsSync(path.join(_INPUT_PATH, filename))) { // If file already downloaded in local directory, use that instead of downloading again 727 | wss.broadcast(JSON.stringify({'event': 'download', 'status': 'complete', 'file': filename})); 728 | return callback(); 729 | 730 | } else { 731 | // TODO: upgrade to aria2c, significantly faster 732 | // Log download times 733 | 734 | var _download_start_time = Date.now(); 735 | 736 | bucket.file(filename).download({ destination: path.join(_INPUT_PATH, filename) }, function(err) { 737 | if(err) { // Error handling (bucket file not found in GCS) 738 | wss.broadcast(JSON.stringify({'event': 'download', 'status': 'error', 'message': err.code + ' ' + err.message})); 739 | return callback(err.code + ' ' + err.message); 740 | } 741 | 742 | request.put({ url: `${_API_HOST}/v2/transcode/jobs/${job.id}/download_meta`, json: { download_time: parseInt(Date.now() - _download_start_time), download_method: 'google-cloud' }}, function(error, response, body) { }); 743 | 744 | wss.broadcast(JSON.stringify({'event': 'download', 'status': 'complete', 'file': filename})); 745 | return callback(); 746 | }); 747 | } 748 | }, 749 | // Clear output directories 750 | function(callback) { 751 | fs.emptyDir(_OUTPUT_PATH, err => { // Clear out output path of old m3u8 files 752 | if (err) return callback(err); 753 | fs.mkdirSync(path.join(_OUTPUT_PATH, 'thumbs')); 754 | callback(); 755 | }); 756 | }, 757 | function(callback) { 758 | async.parallel([ 759 | (callback) => { 760 | _HD_720P = ffmpeg(path.join(_INPUT_PATH, filename), { presets: _PRESETS_PATH }).preset('highlight_mp4') 761 | .videoBitrate(2000) 762 | .audioBitrate('96k') 763 | .renice(-10); 764 | 765 | if(job.parameters.startTime && job.parameters.endTime) { 766 | job.parameters.startTime = Math.max(job.parameters.startTime, 0); 767 | _HD_720P.seekInput(job.parameters.startTime - 10 < 0 ? 0 : job.parameters.startTime - 10); 768 | _HD_720P.outputOptions(['-ss ' + (job.parameters.startTime).toFixed(2), '-t ' + (job.parameters.endTime - job.parameters.startTime), '-copyts', '-start_at_zero']); 769 | } 770 | 771 | if(path.extname(filename) == '.avi') { 772 | _HD_720P.outputOptions('-filter:v', "yadif=0, scale=w=1280:h=720'"); 773 | _HD_720P.outputOptions('-pix_fmt', 'yuv420p'); 774 | } 775 | else _HD_720P.outputOptions('-filter:v', 'scale=w=1280:h=720'); 776 | 777 | _HD_720P.on('start', function(commandLine) { 778 | wss.broadcast(JSON.stringify({'event': 'm3u8', 'status': 'start', 'rendition': '720P_2000K', 'command': commandLine})); 779 | }) 780 | .on('progress', function(progress) { 781 | progress['event'] = 'progress'; 782 | progress['rendition'] = '720P_2000K'; 783 | wss.broadcast(JSON.stringify(progress)); 784 | }) 785 | .on('stderr', function(stderrLine) { 786 | //console.log('Stderr output: ' + stderrLine); 787 | }) 788 | .on('error', function(err, stdout, stderr) { 789 | wss.broadcast(JSON.stringify({'event': 'error', 'message': err.message})); 790 | return callback(err.message); 791 | }) 792 | .on('end', function(stdout, stderr) { 793 | wss.broadcast(JSON.stringify({'event': 'm3u8', 'status': 'complete', 'rendition': '720P_2000K'})); 794 | 795 | return callback(null, uuid + '_720p_3000k.mp4'); 796 | }) 797 | .saveToFile(_OUTPUT_PATH + '/' + uuid + '_720p_3000k.mp4'); 798 | }, 799 | (callback) => { 800 | CREATE_THUMBNAILS(path.join(_INPUT_PATH, filename), job.parameters.startTime, job.parameters.endTime, callback); 801 | } 802 | ], (err, results) => { 803 | if(err) return callback(err); 804 | return callback(null, results); 805 | }); 806 | }, 807 | // Upload to GCS 808 | function(filenames, callback) { 809 | dir.files(_OUTPUT_PATH, function(err, files) { 810 | if(err) throw err; 811 | console.log('Files to upload: ' + files.length); 812 | 813 | async.eachOfLimit(files, 100, function(absolute_fn, key, callback) { 814 | if(!MIME_TYPE_LUT[path.extname(absolute_fn)]) return; // Only upload allowed extensions 815 | 816 | var _options = { // GCS options 817 | resumable: false, // default is true for files >5MB 818 | validation: false, // Disable crc32/md5 checksum validation 819 | destination: path.join(_GCS_BASEPATH, path.relative(_OUTPUT_PATH, absolute_fn)), 820 | metadata: { 821 | contentType: MIME_TYPE_LUT[path.extname(absolute_fn)], 822 | cacheControl: 'public, max-age=31556926' 823 | } 824 | }; 825 | 826 | GCS_UPLOAD_RECURSIVE(absolute_fn, _options, callback); 827 | }, 828 | function(err) { 829 | console.log('Completed upload ' + files.length); 830 | callback(null, filenames); 831 | }); 832 | }); 833 | } 834 | ], function(err, results) { 835 | if(err) { 836 | Raven.captureException(err); 837 | return request.put({ 838 | url: _API_HOST + '/v2/transcode/jobs/' + job.id + '/error', 839 | method: 'PUT', 840 | json: { message: err } 841 | }, function(error, response, body) { 842 | _transcodeInProgress = false; 843 | console.log('PUTTING TO assets_transcode_queue: ERROR') 844 | }); 845 | } 846 | 847 | async.parallel([ 848 | function(callback) { // Update queue status to FINISHED 849 | console.log('PUTTING TO assets_transcode_queue: FINISHED'); 850 | request.put(_API_HOST + '/v2/transcode/jobs/' + job.id + '/finished', function(error, response, body) { callback(); }); 851 | }, 852 | function(callback) { // Update user_exported_highlights table with urls 853 | console.log("PUT TO user_exported_highlights"); 854 | 855 | _PUT_BODY = { 856 | mp4_720p: 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, results[0]), 857 | thumbnails: JSON.stringify(results[1].map(function(e) {return 'https://cdn-google.broadcast.cx/' + path.join(_GCS_BASEPATH, 'thumbs', e)})), 858 | duration: parseInt((job.parameters.endTime - job.parameters.startTime) * 1000) 859 | }; 860 | 861 | 862 | request.put({ 863 | url: _API_HOST + '/v2/assets/individual_highlight/' + job.parameters.individual_highlight_id, 864 | method: 'PUT', 865 | json: _PUT_BODY 866 | }, function(error, response, body) { callback(); }); 867 | } 868 | ], function(err, response) { 869 | _transcodeInProgress = false; 870 | }); 871 | }); 872 | 873 | } 874 | 875 | 876 | 877 | 878 | 879 | setInterval(function() { // Poll DB for new jobs if there is no transcode in progress 880 | if(_transcodeInProgress) return; 881 | 882 | async.waterfall([ 883 | (callback) => { 884 | // Get list of queued transcodes from the DB 885 | // Send machine information along 886 | request.put({ 887 | url: _API_HOST + '/v2/transcode/jobs/request', 888 | method: 'PUT', 889 | json: { 890 | machine_details: machine_details // Send machine information 891 | }, 892 | timeout: 1500 893 | }, (error, response, body) => { 894 | if(error || ![404, 200].includes(response.statusCode)) return callback(`Error connecting to queue (${error || response.statusCode})`); 895 | if(body.length == 0) return callback('No jobs in the queue.'); 896 | 897 | callback(null, body); 898 | }); 899 | }, 900 | (job, callback) => { 901 | _transcodeInProgress = true; 902 | 903 | if(job.type == 'fullGame') MASTER_GAME_FOOTAGE_HLS(job.filename, job, callback); 904 | if(job.type == 'event') MASTER_TRIM(job.filename, job, callback); 905 | if(job.type == 'individualHighlight') MASTER_INDIVIDUAL_HIGHLIGHT(job.filename, job, callback); 906 | } 907 | ], (err, response) => { 908 | if(err) return console.log(err); 909 | }); 910 | }, 2000 + Math.floor(Math.random() * 1000)); // Keep initialization times somewhat random 911 | 912 | 913 | server.listen(_PORT, function() { 914 | console.log('%s listening at %s', server.name, server.url); 915 | }); 916 | 917 | 918 | 919 | 920 | 921 | -------------------------------------------------------------------------------- /index.m3u8: -------------------------------------------------------------------------------- 1 | #EXTM3U 2 | #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=3128000,RESOLUTION=1280x720,CODECS="avc1.640028,mp4a.40.2" 3 | 720p_3000k.m3u8 4 | #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1628000,RESOLUTION=853x480,CODECS="avc1.640028,mp4a.40.2" 5 | 480p_1500k.m3u8 6 | #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=978000,RESOLUTION=640x360,CODECS="avc1.640028,mp4a.40.2" 7 | 360p_850k.m3u8 8 | #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=528000,RESOLUTION=426x240,CODECS="avc1.640028,mp4a.40.2" 9 | 240p_400k.m3u8 10 | #EXT-X-STREAM-INF:PROGRAM-ID=1, BANDWIDTH=128000, CODECS="mp4a.40.2" 11 | 128kbps_aac.m3u8 -------------------------------------------------------------------------------- /keys/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xyk2/cloud-transcoder/f1a52633d2ca6ce3fe98cc1d04ad07a8afc0e806/keys/.keep -------------------------------------------------------------------------------- /mt-downloader-benchmark.js: -------------------------------------------------------------------------------- 1 | var url = require("url") 2 | var path = require("path") 3 | var mtd = require('zeltice-mt-downloader') 4 | 5 | var target_url = "http://storage.googleapis.com/broadcast-cx-raw-recordings/8dc9_F70D11E4-8ADE-46B4-BE4A-4A791CA51166.MOV" 6 | var file_name = path.basename(url.parse(target_url).pathname) 7 | var file_path = path.join(__dirname, "8dc9_F70D11E4-8ADE-46B4-BE4A-4A791CA51166.MOV") 8 | 9 | var start_time = null; 10 | var mt_downloading = null; 11 | 12 | var downloader = new mtd(file_path, target_url, { 13 | count: 8, // (Default: 2) 14 | method: 'GET', // (Default: GET) 15 | port: 80, // (Default: 80) 16 | timeout: 5, // (Default: 5 seconds) 17 | onStart: function(meta) { 18 | console.log('Download started mt-downloader...'); 19 | start_time = Date.now(); 20 | 21 | mt_downloading = true; 22 | 23 | setInterval(function() { 24 | if(mt_downloading) { 25 | for(var x in meta.threads) { 26 | console.log(meta.threads[x].start / 1000, meta.threads[x].end / 1000, meta.threads[x].position / 1000); 27 | } 28 | } 29 | }, 2000); 30 | }, 31 | 32 | //Triggered when the download is completed 33 | onEnd: function(err, result) { 34 | if (err) console.error(err); 35 | else console.log('Download complete mt-downloader.'); 36 | mt_downloading = false; 37 | console.log((Date.now() - start_time)/1000); 38 | 39 | 40 | var google_cloud = require('google-cloud')({ 41 | projectId: 'broadcast-cx', 42 | keyFilename: 'broadcast-cx-bda0296621a4.json' 43 | }); 44 | 45 | var gcs = google_cloud.storage(); 46 | var bucket = gcs.bucket('broadcast-cx-raw-recordings'); 47 | 48 | start_time = Date.now(); 49 | console.log('Download started google-cloud...'); 50 | 51 | bucket.file('8dc9_F70D11E4-8ADE-46B4-BE4A-4A791CA51166.MOV').download({ 52 | destination: 'GCP_8dc9_F70D11E4-8ADE-46B4-BE4A-4A791CA51166.MOV' 53 | }, function(err) { 54 | 55 | console.log('Download complete google-cloud.'); 56 | console.log((Date.now() - start_time)/1000); 57 | }); 58 | } 59 | }); 60 | 61 | downloader.start(); -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Choxue-ffmpeg", 3 | "version": "1.0.0", 4 | "description": "VM instance to do splitting / combining, transcoding, highlights generation.", 5 | "main": "app.js", 6 | "author": "Xiao Yang Kao", 7 | "license": "ISC", 8 | "dependencies": { 9 | "async": "^2.5.0", 10 | "fluent-ffmpeg": "^2.1.2", 11 | "fs-extra": "^2.1.2", 12 | "google-cloud": "^0.56.0", 13 | "m3u8-parser": "^3.0.0", 14 | "node-dir": "^0.1.17", 15 | "pg": "^6.4.2", 16 | "raven": "^2.2.1", 17 | "request": "^2.80.0", 18 | "restify": "^4.3.1", 19 | "restify-plugins": "^1.6.0", 20 | "uuid": "^3.1.0", 21 | "ws": "^2.3.1" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /presets/highlight_mp4.js: -------------------------------------------------------------------------------- 1 | exports.load = function(ffmpeg) { 2 | ffmpeg 3 | .outputOptions('-preset', 'fast') 4 | .outputOptions('-profile:v', 'high') 5 | .outputOptions('-level', '4.0') 6 | .outputOptions('-movflags', '+faststart') 7 | }; -------------------------------------------------------------------------------- /presets/hls.js: -------------------------------------------------------------------------------- 1 | exports.load = function(ffmpeg) { 2 | ffmpeg 3 | .outputOptions('-c:a', 'aac') 4 | .outputOptions('-preset', 'veryfast') 5 | .outputOptions('-x264-params', 'keyint=60:min-keyint=60:scenecut=-1') 6 | .outputOptions('-profile:v', 'high') 7 | .outputOptions('-level', '4.0') 8 | //.outputOptions('-ac', '2') 9 | .outputOptions('-movflags', '+faststart') 10 | .outputOptions('-start_number', '0') 11 | .outputOptions('-hls_time', '6') 12 | .outputOptions('-hls_list_size', '0') 13 | .outputOptions('-max_muxing_queue_size', '1024') // Some MP4s coming from Wowza error out on start because buffer is too small 14 | .outputOptions('-f', 'hls'); 15 | }; -------------------------------------------------------------------------------- /presets/m3u8_to_mp4.js: -------------------------------------------------------------------------------- 1 | exports.load = function(ffmpeg) { 2 | ffmpeg 3 | .outputOptions('-c', 'copy') 4 | .outputOptions('-bsf:a', 'aac_adtstoasc') 5 | .outputOptions('-movflags', '+faststart') 6 | }; -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Cloud transcoding adaptive bitrate MP4 and HLS 2 | 3 | Transcodes a source video file (MP4, AVI, MPG, M4A, WMV, MOV) to adaptive bitrate MP4 and HLS to be stored in Google Cloud Storage. Cheaper than 3rd party transcoding services by more than 40-50x when running on preemptible instances (compared to AWS Elastic Transcoder, Zencoder, Bitmovin, Cloudinary), and saves time/bandwidth massively by transcoding in the same region as where the files are stored. 4 | 5 | 6 | ### Configuring boot disk template for GCE 7 | ```bash 8 | sudo add-apt-repository ppa:jonathonf/ffmpeg-3 # Enter to confirm 9 | sudo apt-get update 10 | sudo apt-get -y upgrade 11 | sudo apt install -y ffmpeg libav-tools x264 x265 12 | 13 | sudo curl -sL https://deb.nodesource.com/setup_6.x -o nodesource_setup.sh 14 | sudo bash nodesource_setup.sh 15 | sudo apt-get install nodejs -y 16 | sudo npm install pm2 -g 17 | 18 | # Route 8080 to port 80 to allow node to access port 80 without root privileges 19 | sudo iptables -t nat -I PREROUTING -p tcp --dport 80 -j REDIRECT --to-port 8080 20 | 21 | # Create ROOT RSA private key then copy to Github repo (pull access only) 22 | sudo ssh-keygen -t rsa 23 | sudo cat /root/.ssh/id_rsa.pub 24 | ``` 25 | 26 | 27 | ### Startup script for each new instance 28 | ```bash 29 | #! /bin/bash 30 | 31 | ssh-keyscan github.com >> ~/.ssh/known_hosts 32 | git clone git@github.com:xyk2/cloud-transcoder.git 33 | 34 | cd /cloud-transcoder 35 | npm install 36 | PM2_HOME=/root/.pm2 pm2 kill 37 | PM2_HOME=/root/.pm2 NODE_ENV=production pm2 start app.js 38 | ``` 39 | 40 | ### Shutdown script before preemption 41 | * Call localhost endpoint to reset running job in DB queue 42 | * Terminate PM2 43 | * 44 | 45 | ### Workflow to upload & transcode & tag game videos from YouTube 46 | * `youtube-dl` to download and merge original files 47 | * `gsutil -m cp` to `broadcast-cx-raw-recordings` bucket 48 | * Insert filename, size, and format into `assets_game_footage` table 49 | * Insert type, id, status into `assets_transcode_queue` 50 | * Find game ID of game film, insert into `assets_game_footage` which will be populated 51 | * Find start_record_time of game film, insert into `assets_game_footage` 52 | 53 | 54 | ### Useful snippets 55 | `youtube-dl` 56 | youtube-dl --external-downloader aria2c --external-downloader-args '-x 8' https://www.youtube.com/playlist?list=PLUz3zvlwsLgWJRmJo8fLOTMiXkFC25S61 --playlist-start 46 --playlist-end 150 57 | 58 | 59 | ### ffmpeg errors and possible errors 60 | * `ffmpeg exited with code 1: Conversion failed!` -max_muxing_queue_size 1024 61 | 62 | 63 | 64 | 65 | 66 | --------------------------------------------------------------------------------