HLS Manager

├── .gitignore ├── 1.sh ├── Procfile ├── README.md ├── Site ├── css │ └── app.css ├── index.html └── js │ └── app.js ├── hlsGrep.js ├── hlsPlay.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # Compiled binary addons (http://nodejs.org/api/addons.html) 20 | build/Release 21 | 22 | # Dependency directory 23 | # Commenting this out is preferred by some people, see 24 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 25 | node_modules 26 | 27 | # Users Environment Variables 28 | .lock-wscript 29 | 30 | #idea 31 | .idea 32 | 33 | public 34 | public_tmp 35 | 36 | 37 | -------------------------------------------------------------------------------- /1.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kaltura/hls-toolkit/150f72e6db0f2dc2c915680c5bbaeec800227723/1.sh -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: node hlsPlay.js -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hls-toolkit 2 | 3 | ## Record HLS stream: 4 | node hlsGrep.js [Stream url.m3u8] [StreamName] 5 | 6 | ## Replay the stream: 7 | node hlsPlay.js 8 | 9 | from the hls player open http://localhost:6060/[StreamName]/play.m3u8 10 | -------------------------------------------------------------------------------- /Site/css/app.css: -------------------------------------------------------------------------------- 1 | .blink_me { 2 | -webkit-animation-name: blinker; 3 | -webkit-animation-duration: 2s; 4 | -webkit-animation-timing-function: linear; 5 | -webkit-animation-iteration-count: infinite; 6 | 7 | -moz-animation-name: blinker; 8 | -moz-animation-duration: 2s; 9 | -moz-animation-timing-function: linear; 10 | -moz-animation-iteration-count: infinite; 11 | 12 | animation-name: blinker; 13 | animation-duration: 2s; 14 | animation-timing-function: linear; 15 | animation-iteration-count: infinite; 16 | } 17 | 18 | @-moz-keyframes blinker { 19 | 0% { opacity: 1.0; } 20 | 50% { opacity: 0.0; } 21 | 100% { opacity: 1.0; } 22 | } 23 | 24 | @-webkit-keyframes blinker { 25 | 0% { opacity: 1.0; } 26 | 50% { opacity: 0.0; } 27 | 100% { opacity: 1.0; } 28 | } 29 | 30 | @keyframes blinker { 31 | 0% { opacity: 1.0; } 32 | 50% { opacity: 0.0; } 33 | 100% { opacity: 1.0; } 34 | } -------------------------------------------------------------------------------- /Site/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | HLS Manager 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 |

15 | 23 | 40 | 41 |

Stream capture complete/ready

42 |

Capture stream in progress

43 |
44 |
45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 |

Stream Name	URL	Actions	Bitrates
{{stream.name}}	{{stream.url}}		{{stream.numOfBitrate}}

65 | 66 | 67 | 68 |
69 |
70 | 71 | Capture logs 72 | 75 |

76 | 77 | 78 | -------------------------------------------------------------------------------- /Site/js/app.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by itayk on 4/19/15. 3 | */ 4 | angular.module('hlsManager', []) 5 | .controller('hlsManagerController', function($http,$scope) { 6 | var hlsManager = this; 7 | hlsManager.captureLogs=false; 8 | hlsManager.progress = true; 9 | hlsManager.reset = function(){ 10 | $http.get('/reset' ); 11 | }; 12 | hlsManager.add = function(){ 13 | var hlsURL = encodeURIComponent(hlsManager.hlsURL); 14 | var streamName = hlsManager.streamName; 15 | var isLive = hlsManager.isLive === true?true:false; 16 | hlsManager.progressInterval = setInterval(function(){ 17 | hlsManager.isProgress(); 18 | },1000); 19 | $http.get('/add/'+streamName+'/'+hlsURL +'/'+isLive); 20 | }; 21 | hlsManager.isProgress = function(){ 22 | $http.get('/progress' ).success(function(data){ 23 | if (!data){ 24 | clearInterval(hlsManager.progressInterval); 25 | getStreams(); 26 | } 27 | hlsManager.progress = data; 28 | }) 29 | }; 30 | hlsManager.kill = function(){ 31 | $http.get('/kill'); 32 | }; 33 | hlsManager.clearLogs = function(){ 34 | hlsManager.logs = ''; 35 | 36 | } 37 | hlsManager.streams = [ 38 | {name: "Stream1" , url: "http://xxxx/1.m3u8"} , 39 | {name: "Stream2" , url: "http://xxxx/2.m3u8"} 40 | ]; 41 | hlsManager.playVOD = function(){ 42 | $http.get('/playVOD') ; 43 | }; 44 | hlsManager.playLive = function(){ 45 | $http.get('/playLive') ; 46 | }; 47 | hlsManager.disableBitrate = function(stream){ 48 | var streamName = stream.stream.name; 49 | var bitrate = this.selectedBitRate; 50 | $http.get('/disableTrack/'+streamName+'/' + bitrate); 51 | }; 52 | hlsManager.resetBitRates = function(stream){ 53 | 54 | $http.get('/resetDisableTrack'); 55 | }; 56 | var getStreams = function() { 57 | $http.get( '/list' ).success( function ( data ) { 58 | hlsManager.streams = data; 59 | } ); 60 | }; 61 | setInterval(function(){ 62 | if( hlsManager.captureLogs) { 63 | $http.get( '/logs' ).success( function ( data ) { 64 | hlsManager.logs = data +hlsManager.logs ; 65 | } ); 66 | } 67 | },1000); 68 | getStreams(); 69 | hlsManager.isProgress(); 70 | }); 71 | 72 | -------------------------------------------------------------------------------- /hlsGrep.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by itayk on 2/11/15. 3 | */ 4 | var fs = require('fs'); 5 | var http = require('http'); 6 | var q = require('q'); 7 | var request = require('request'); 8 | var arguments = process.argv.slice(2); 9 | var hlsStream = arguments[0] || 'http://abclive.abcnews.com/i/abc_live4@136330/master.m3u8'; 10 | var name= arguments[1] || 'ABC'; 11 | var baseDir = 'public/' + name + 'Stream/'; 12 | if ( !fs.existsSync('public')){ 13 | fs.mkdirSync( 'public' ); 14 | } 15 | if ( !fs.existsSync(baseDir) ) { 16 | fs.mkdirSync( baseDir ); 17 | } 18 | var masterFile = fs.createWriteStream(baseDir+ 'master.m3u8'); 19 | 20 | function log(text){ 21 | var now = new Date().getTime(); 22 | console.log(now + "HLSGrep ---- " , text); 23 | } 24 | 25 | function readMaster(){ 26 | var deferred = q.defer(); 27 | request.get(hlsStream, function (error, response, body) { 28 | console.log("Master downloaded."); 29 | deferred.resolve(); 30 | }).pipe(masterFile); 31 | 32 | return deferred.promise; 33 | } 34 | 35 | function parseMaster(){ 36 | var defferred = q.defer(); 37 | var bw = {}; 38 | var readFile = fs.readFile(baseDir+ 'master.m3u8','utf8',function(err,data){ 39 | log(data); 40 | var lines = data.split('\n'); 41 | for (var i=0;i 1){ 45 | bw[bwMatch[1]] = lines[i+1]; 46 | } 47 | } 48 | defferred.resolve(bw); 49 | 50 | }); 51 | return defferred.promise; 52 | } 53 | 54 | function downloadSegments(bwObj){ 55 | 56 | if ( !fs.existsSync(name + 'Stream')) { 57 | fs.mkdirSync( name + 'Stream' ); 58 | } 59 | var jobs = []; 60 | for (var i in bwObj){ 61 | var path = baseDir + 'bitRate_'+i +'/'; 62 | if ( !fs.existsSync(path)) { 63 | fs.mkdirSync( path); 64 | } 65 | jobs.push({arg1:bwObj[i],arg2:path}); 66 | // monitorAndDownload(bwObj[i],path ); 67 | // break; 68 | } 69 | var x = jobs.pop(); 70 | var worker = function(){ 71 | x = jobs.pop(); 72 | return monitorAndDownload( x.arg1, x.arg2 ); 73 | }; 74 | var qq = monitorAndDownload( x.arg1, x.arg2 ); 75 | for (var i=0;i< jobs.length ; i++){ 76 | qq = qq.then(worker); 77 | } 78 | } 79 | 80 | function monitorAndDownload(url,path){ 81 | q = require('q'); 82 | var deferred = q.defer(); 83 | var numOfFiles = 0; 84 | var timeout = 30; 85 | var queue = []; 86 | if (url.indexOf("http") == -1){ 87 | url = hlsStream.replace(/([\w,\s-]+\.m3u8)/ig,url); 88 | } 89 | request(url, function (error, response, body) { 90 | if (!error && response.statusCode == 200) { 91 | var fileExsit = false; 92 | var passheader = false; 93 | if (!fs.existsSync(path +'playlist.m3u8')) { 94 | fs.writeFile( path + 'playlist.m3u8','' ); 95 | } 96 | else{ 97 | fileExsit = true; 98 | } 99 | var tsHash = {}; 100 | var lines = body.split('\n'); 101 | for (var i=0;i 1){ 106 | 107 | request.get(url.replace(/([\w,\s-]+\.m3u8)/ig,keyMatch[1]) ).on( 'error' , function ( err ) { 108 | log( err ) 109 | } ) 110 | .pipe( fs.createWriteStream( path + keyMatch[1] ) ); 111 | } 112 | 113 | var tsLength = line.match(/#EXTINF:([0-9\.]*)/); 114 | if (tsLength && tsLength.length>1){ 115 | passheader = true; 116 | var fileName = lines[i+1].match(/([\w,\s-]+\.ts)/); 117 | if (fileName && fileName.length>1) { 118 | if (!fs.existsSync(path + fileName[1])) { 119 | fs.appendFileSync(path +'playlist.m3u8',line + "\n"); 120 | fs.appendFileSync(path +'playlist.m3u8',fileName[1] + "\n"); 121 | var tsUrl = lines[i+1]; 122 | if (tsUrl.indexOf("http") == -1){ 123 | tsUrl = url.replace(/([\w,\s-]+\.m3u8)/ig,tsUrl); 124 | } 125 | numOfFiles++; 126 | queue.push({url:tsUrl,path:path+fileName[1]}); 127 | } 128 | } 129 | }else{ 130 | if (!fileExsit && !passheader) { 131 | fs.appendFileSync( path + 'playlist.m3u8' , line + "\n" ); 132 | } 133 | } 134 | } 135 | 136 | var worker = function(){ 137 | q = require('q'); 138 | var deferred2 = q.defer(); 139 | log("Grabbing file:" + queue.length); 140 | if (queue.length == 0 ){ 141 | deferred2.resolve(); 142 | 143 | } else { 144 | var item = queue.pop(); 145 | request.get( item.url ) 146 | .on( 'error' , function ( err ) { 147 | deferred2.reject(); 148 | log( err ) 149 | } ) 150 | .on( 'response' , function ( res ) { 151 | numOfFiles--; 152 | deferred2.resolve(); 153 | } ) 154 | .pipe( fs.createWriteStream( item.path ) ); 155 | } 156 | return deferred2.promise; 157 | }; 158 | var qqq = worker(); 159 | for (var i=0;i 0 && timeout > 0){ 169 | log("Sleeping for 5 sec"); 170 | sleep(); 171 | } else { 172 | deferred.resolve(numOfFiles); 173 | } 174 | 175 | }, 5000);}; 176 | sleep(); 177 | return deferred.promise; 178 | } 179 | 180 | readMaster() 181 | .then( parseMaster) 182 | .then( downloadSegments); 183 | 184 | -------------------------------------------------------------------------------- /hlsPlay.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by itayk on 2/12/15. 3 | */ 4 | var express = require('express'); 5 | var fs = require('fs'); 6 | var q= require('q'); 7 | var app = express(); 8 | var streamFolder = 'public'; 9 | var startTime = Date.now(); 10 | var diffTime = 20; 11 | var windowSize = 5; 12 | var numOfLiveSameple = 10; 13 | var inProgress = false; 14 | var exec = null; 15 | var isVOD = false; 16 | var logs = ''; 17 | var disabledRates = {}; 18 | if ( !fs.existsSync(streamFolder)){ 19 | console.error("Can't find public folder"); 20 | fs.mkdirSync('public'); 21 | //process.exit(1); 22 | } 23 | 24 | function log(text){ 25 | var now = new Date().getTime(); 26 | console.log(now + "HLSPlay ---- " , text); 27 | if ( typeof text != 'string') { 28 | try{ 29 | text = JSON.stringify(text); 30 | } catch(e){} 31 | } 32 | logs = now + "HLSPlay ---- " + text + "\\\n" + logs; 33 | } 34 | 35 | app.use(express.static(__dirname + '/' + streamFolder)); 36 | app.use(express.static(__dirname + '/Site' )); 37 | 38 | function parsePlaylist(data,folder){ 39 | var resultObject = []; 40 | var lines = data.split('\n'); 41 | for (var i=0 ; i1) { 45 | resultObject.push(currentLine + '\n' + folder +'/' + lines[i+1]+'\n'); 46 | } 47 | } 48 | return resultObject; 49 | } 50 | 51 | function scanForStreams(){ 52 | var resultObj = {}; 53 | var streams = fs.readdirSync(streamFolder); 54 | streams.forEach(function(item,index){ 55 | 56 | if (fs.lstatSync(streamFolder +'/' + item ).isDirectory()) { 57 | resultObj[item] = []; 58 | var bitRates = fs.readdirSync( streamFolder + '/' + item ); 59 | bitRates.forEach( function ( bitem , bindex ) { 60 | if (fs.lstatSync(streamFolder + '/' + item + '/' + bitem ).isDirectory()) { 61 | var bitrate = bitem.split( "_" ); 62 | if ( bitrate.length > 1 ) { 63 | bitrate = bitrate[1]; 64 | var tsList = {}; 65 | tsList.bitrate =bitrate; 66 | tsList.count=100; 67 | tsList.duration = 1000; 68 | tsList.data = parsePlaylist(fs.readFileSync( streamFolder + '/' + item + '/' + bitem + '/playlist.m3u8' , 'utf8' ),bitem); 69 | resultObj[item].push(tsList); 70 | } 71 | } 72 | } ); 73 | } 74 | log(resultObj); 75 | }); 76 | return resultObj; 77 | } 78 | var streams = scanForStreams(); 79 | 80 | app.get('/reset',function(req, res, next){ 81 | startTime = Date.now(); 82 | res.send('Time reset'); 83 | }); 84 | app.get('/:stream/play.m3u8', function(req, res, next) { 85 | log(req.url); 86 | var streamName = req.params['stream']; 87 | if ( !streamName && !streams[streamName] ) { 88 | next(); 89 | return; 90 | } 91 | //res.send('found!'); 92 | var response = ''; 93 | var masterFile = fs.readFileSync(streamFolder + '/' +streamName + '/master.m3u8', 'utf8' ).split('\n'); 94 | for (var i=0 ; i < masterFile.length; i++){ 95 | var currentLine = masterFile[i]; 96 | if (currentLine.indexOf('#') == 0){ 97 | response += currentLine +'\n'; 98 | } else { 99 | var preLine = masterFile[i-1]; 100 | var bwMatch = preLine.match(/#EXT-.*BANDWIDTH=([0-9]*),/); 101 | if (bwMatch && bwMatch.length >1) { 102 | response += '/'+ streamName +'/bitRate_' + bwMatch[1] + '.m3u8\n'; 103 | } 104 | } 105 | } 106 | res.contentType('application/vnd.apple.mpegurl'); 107 | res.send(response); 108 | 109 | }); 110 | 111 | app.get('/crossdomain.xml' ,function(req, res, next) { 112 | res.send('

'); 113 | }); 114 | 115 | app.get('/:stream/bitrate_:rate.m3u8',function(req, res, next){ 116 | log(req.url); 117 | 118 | var response = '#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-ALLOW-CACHE:NO\n#EXT-X-TARGETDURATION:13\n#EXT-X-MEDIA-SEQUENCE:0\n'; 119 | var streamName = req.params['stream']; 120 | var bitRate = req.params['rate']; 121 | if (disabledRates[streamName]){ 122 | for (var i=0; i1) { 150 | timeTillNow += parseFloat( tsLength[1] ); 151 | if (timeTillNow < diff + diffTime || isVOD ){ 152 | response += currentLine; 153 | } 154 | } 155 | } 156 | } 157 | if (isVOD) 158 | response += '#EXT-X-ENDLIST'; 159 | res.contentType('application/vnd.apple.mpegurl'); 160 | 161 | res.send(response); 162 | 163 | }); 164 | 165 | app.get('/list' , function(req, res, next){ 166 | streams = scanForStreams(); 167 | var result = []; 168 | for (var i in streams){ 169 | result.push({name:i,url:"/"+i+"/play.m3u8",numOfBitrate:streams[i].length}); 170 | } 171 | res.send(JSON.stringify(result)); 172 | }); 173 | 174 | app.get('/add/:name/:url/:isLive', function(req,res,next){ 175 | log(req.url); 176 | if (inProgress){ 177 | res.send("Error - Wait for the current capture to end :-)"); 178 | return; 179 | } 180 | inProgress = true; 181 | var streamName = req.params["name"]; 182 | var streamURL = decodeURIComponent(req.params["url"]); 183 | var live = req.params["isLive"]; 184 | var samepleCount = req.params["SampleCount"]; 185 | if (live === "true"){ 186 | numOfLiveSameple = 10; 187 | if (samepleCount && parseInt(samepleCount)) { 188 | numOfLiveSameple = parseInt(samepleCount); 189 | } 190 | } 191 | var executeGrep = function() { 192 | exec = require( 'child_process' ).exec; 193 | exec( 'node hlsGrep ' + streamURL + ' ' + streamName , function callback( error , stdout , stderr ) { 194 | if (live === "true" && numOfLiveSameple > 0){ 195 | log("Grepping live content " + numOfLiveSameple +" to go"); 196 | numOfLiveSameple--; 197 | executeGrep(); 198 | return; 199 | } 200 | inProgress = false; 201 | log( stdout ); 202 | } ); 203 | }; 204 | executeGrep(); 205 | }); 206 | 207 | app.get('/progress' , function(req,res,next){ 208 | res.send(inProgress); 209 | }); 210 | 211 | app.get('/kill' , function(req,res,next){ 212 | if (exec) { 213 | exec.kill( 'kill' ); 214 | inProgress = false; 215 | } 216 | }); 217 | 218 | app.get('/logs', function(req,res,next){ 219 | res.send(logs); 220 | logs = ''; 221 | }); 222 | 223 | app.get('/playVOD', function(req,res,next){ 224 | isVOD = true; 225 | res.send( true ); 226 | }); 227 | 228 | app.get('/playLive', function(req,res,next){ 229 | isVOD = false; 230 | res.send( true ); 231 | }); 232 | 233 | app.get('/playMode', function(req,res,next) { 234 | res.send( isVOD ); 235 | }); 236 | 237 | app.get('/disableTrack/:name/:bitRate', function(req,res,next) { 238 | var streamName = req.params["name"]; 239 | var bitRate = req.params["bitRate"]; 240 | disabledRates[streamName] = disabledRates[streamName] || []; 241 | disabledRates[streamName].push(bitRate); 242 | res.send( true ); 243 | 244 | }); 245 | app.get('/resetDisableTrack', function(req,res,next) { 246 | disabledRates = {}; 247 | res.send( true ); 248 | }); 249 | 250 | app.use(function(req, res, next) { 251 | var err = new Error('Not Found'); 252 | err.status = 404; 253 | next(err); 254 | }); 255 | 256 | app.use(function(err, req, res, next) { 257 | res.status(err.status); 258 | res.send({message: err.message}); 259 | }); 260 | 261 | app.listen(process.env.PORT || 8080); 262 | 263 | module.exports = app; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hls-toolkit", 3 | "version": "0.0.1", 4 | "description": "Record&Replay HLS streams", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/kaltura/hls-toolkit.git" 12 | }, 13 | "keywords": [ 14 | "HLS", 15 | "apple" 16 | ], 17 | "dependencies": { 18 | "express": "~4.9.x" , 19 | "q":"~1.1.2", 20 | "request":"~2.53.0" 21 | }, 22 | "author": "Itay Kinnrot", 23 | "license": "ISC", 24 | "bugs": { 25 | "url": "https://github.com/kaltura/hls-toolkit/issues" 26 | }, 27 | "homepage": "https://github.com/kaltura/hls-toolkit" 28 | } 29 | --------------------------------------------------------------------------------