├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── bin └── live-stream ├── lib ├── config.js ├── live-stream-controller.js ├── packager.js ├── reader.js ├── transcoder.js └── watcher.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | 3 | *.ts 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## test-engine-live-tools 1.0.1 (2015/03/30) ## 2 | 3 | * Removed unused package async. 4 | 5 | ## test-engine-live-tools 1.0.0 (2014/11/27) ## 6 | 7 | * Initial release -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, EBU (http://ebu.io/), Hiro (http://madebyhiro.com/). 2 | 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # test-engine-live-tools 2 | 3 | Small tools and scripts for the EBU test engine platform. These allow for DASH-ing and encoding a live stream. 4 | 5 | # Installation 6 | 7 | The easiest way to install is to use the `npm` package. This will give you the latest released version and install all 8 | dependencies (except ffmpeg and MP4Box, you need to provide these yourself): 9 | 10 | # npm install test-engine-live-tools 11 | 12 | If you wish to use a different version from the Git repository you have to clone the code and install the dependencies using npm. 13 | Also, be sure to provide your own MP4Box and ffmpeg binaries, they are not included. 14 | 15 | # git clone https://github.com/ebu/test-engine-live-tools.git 16 | # cd test-engine-live-tools 17 | # npm install 18 | 19 | # Usage 20 | 21 | In general, the tool operates by looping an MPEG-2 Transport Stream and providing that to ffmpeg using the segment muxer. 22 | The generated segments are picked up and packaged/fragmented using MP4Box. After all segments for the desired representations 23 | are available they are DASH-ed by MP4Box using the `--dash-ctx` option. This generates the MPD for the live stream and keeps 24 | it updated. 25 | 26 | A command-line utility is included which allows you to easily stream from the command-line. Usage: 27 | 28 | # bin/live-stream [-c config_file] input_file 29 | 30 | The `config_file` is optional and allows you to create custom settings for your live stream without having to change anything 31 | in the sources. The `input_file` is mandatory and it is required that this is currently muxed as a MPEG-2 Transport Stream for 32 | easy looping of the source material. The contents of the file can be either MPEG-2 video/audio or H264/AAC, or most likely 33 | anything else that ffmpeg can extract from a MPEG-2 TS container. 34 | 35 | ## Using different sources 36 | 37 | Instead of using a looped MPEG-2 TS file directly, it is also possible to read from other input sources. By specifying an input 38 | source using a URL with a protocol, this will be directly passed to ffmpeg to read from that source. This way you can 39 | for example ingest video using UDP transport. Example usage: 40 | 41 | # bin/live-stream udp://192.168.0.10:1234 42 | 43 | In theory any transport protocol supported by your version of ffmpeg is allowed. 44 | 45 | # Configuration 46 | 47 | The default configuration generates one video representation and one audio representation. See `lib/config.js` for details. 48 | All parameter configuration is listed as an array of arguments which NodeJS understands. Custom configuration can be created 49 | by creating a valid JSON file containing an object that overrides values of the default configuration. The complete default configuration is: 50 | 51 | ```javascript 52 | { 53 | segmentDir: '/tmp/dash_segment_input_dir', 54 | outputDir: '/tmp/dash_output_dir', 55 | mp4box: 'MP4Box', 56 | ffmpeg: 'ffmpeg', 57 | encoding: { 58 | commandPrefix: [ '-re', '-i', '-', '-threads', '0', '-y' ], 59 | representations: { 60 | audio: [ 61 | '-map', '0:1', '-vn', '-acodec', 'aac', '-strict', '-2', '-ar', '48000', '-ac', '2', 62 | '-f', 'segment', '-segment_time', '4', '-segment_format', 'mpegts' 63 | ], 64 | video: [ 65 | '-map', '0:0', '-vcodec', 'libx264', '-vprofile', 'baseline', '-preset', 'veryfast', 66 | '-s', '640x360', '-vb', '512k', '-bufsize', '1024k', '-maxrate', '512k', 67 | '-level', '31', '-keyint_min', '25', '-g', '25', '-sc_threshold', '0', '-an', 68 | '-bsf', 'h264_mp4toannexb', '-flags', '-global_header', 69 | '-f', 'segment', '-segment_time', '4', '-segment_format', 'mpegts' 70 | ] 71 | } 72 | }, 73 | packaging: { 74 | mp4box_opts: [ 75 | '-dash-ctx', '/tmp/dash_output_dir/dash-live.txt', '-dash', '4000', '-rap', '-ast-offset', '12', 76 | '-no-frags-default', '-bs-switching', 'no', '-min-buffer', '4000', '-url-template', '-time-shift', 77 | '1800', '-mpd-title', 'MPEG-DASH live stream', '-mpd-info-url', 'http://ebu.io/', '-segment-name', 78 | 'live_$RepresentationID$_', '-out', '/tmp/dash_output_dir/live', '-dynamic', '-subsegs-per-sidx', '-1' 79 | ] 80 | } 81 | } 82 | ``` 83 | 84 | You can use this as a basis for your own configuration. Both the ffmpeg command and MP4Box commands are generated 85 | using this configuration. You can add extra representations by overriding the `encoding` object. 86 | 87 | Please not that some options are required to be able to create a valid live stream. For example: it is required that 88 | we read from stdin using ffmpeg, so `-i -` is required. Also it is preferred to read realtime, so `-re` is also needed. 89 | 90 | The full ffmpeg command is generated by concatenating `encoding.commandPrefix` with the configurations in 91 | `encoding.represenations`. Output files will be automatically added and you should not specify those yourself. 92 | 93 | More example configuration will be added add a later stage. 94 | 95 | # Caveats / pitfalls 96 | 97 | Live streaming using MPEG-DASH is not always easy. To make sure that you are compatible with most DASH clients take extra 98 | care to make sure you're always generating closed GOPs, fixed segment durations and constant bit rates and that timing 99 | over segments is continuous and identical across representations. The default configuration should do just that, but be 100 | sure to keep this in mind. 101 | 102 | # Requirements 103 | 104 | * NodeJS 0.10.x / npm 105 | * ffmpeg binary compiled to taste (recommended: 2.3 or higher) 106 | * MP4Box binary compiled to taste (recommended: r5400 or newer) 107 | 108 | # License 109 | 110 | Available under the BSD 3-clause license. 111 | -------------------------------------------------------------------------------- /bin/live-stream: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | var minimist = require('minimist'); 3 | var liveStreamController = require('../lib/live-stream-controller'); 4 | 5 | function printUsage() { 6 | console.log('usage: live-stream [-h] [-c config_file] input_file'); 7 | console.log(' -h: show this help message'); 8 | console.log(' -c config_file: specify a config file to use, must be a valid JSON file'); 9 | console.log(' input_file: the input file to stream, must be a MPEG-2 transport stream'); 10 | } 11 | 12 | var argv = minimist(process.argv.slice(2), { boolean: ['h'] }); 13 | 14 | if (argv._.length !== 1 || argv.h === true) { 15 | printUsage(); 16 | } else { 17 | var inputFile = argv._[0]; 18 | var configFile = argv.c; 19 | liveStreamController.launch(inputFile, configFile); 20 | } -------------------------------------------------------------------------------- /lib/config.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | var fs = require('fs'); 3 | 4 | var loadedConfig = undefined; 5 | 6 | var defaultConfig = { 7 | inputFile: undefined, 8 | segmentDir: path.sep + path.join('tmp', 'dash_segment_input_dir'), 9 | outputDir: path.sep + path.join('tmp', 'dash_output_dir'), 10 | mp4box: 'MP4Box', 11 | ffmpeg: 'ffmpeg' 12 | } 13 | 14 | var defaultEncoding = { 15 | commandPrefix: [ '-re', '-i', '-', '-threads', '0', '-y' ], 16 | representations: { 17 | audio: [ 18 | '-map', '0:1', '-vn', '-acodec', 'aac', '-strict', '-2', '-ar', '48000', '-ac', '2', 19 | '-f', 'segment', '-segment_time', '4', '-segment_format', 'mpegts' 20 | ], 21 | video: [ 22 | '-map', '0:0', '-vcodec', 'libx264', '-vprofile', 'baseline', '-preset', 'veryfast', 23 | '-s', '640x360', '-vb', '512k', '-bufsize', '1024k', '-maxrate', '512k', 24 | '-level', '31', '-keyint_min', '25', '-g', '25', '-sc_threshold', '0', '-an', 25 | '-bsf', 'h264_mp4toannexb', '-flags', '-global_header', 26 | '-f', 'segment', '-segment_time', '4', '-segment_format', 'mpegts' 27 | ] 28 | } 29 | } 30 | 31 | var defaultPackaging = { 32 | mp4box_opts: [ 33 | '-dash-ctx', path.join(defaultConfig.outputDir, 'dash-live.txt'), '-dash', '4000', '-rap', '-ast-offset', '12', 34 | '-no-frags-default', '-bs-switching', 'no', '-min-buffer', '4000', '-url-template', '-time-shift', '1800', 35 | '-mpd-title', 'MPEG-DASH live stream', '-mpd-info-url', 'http://ebu.io/', '-segment-name', 36 | 'live_$RepresentationID$_', '-out', path.join(defaultConfig.outputDir, 'live'), 37 | '-dynamic', '-subsegs-per-sidx', '-1' 38 | ] 39 | } 40 | 41 | defaultConfig.encoding = defaultEncoding; 42 | defaultConfig.packaging = defaultPackaging; 43 | 44 | function getRepresentationKeys() { 45 | return Object.keys(defaultConfig.encoding.representations); 46 | } 47 | 48 | function getRepresentationKeyForFile(segment) { 49 | var keys = getRepresentationKeys(); 50 | var result = undefined; 51 | keys.forEach(function(key) { 52 | var file = segment.split(path.sep).pop(); 53 | if (file.indexOf(key) == 0) { 54 | result = key; 55 | } 56 | }) 57 | return result; 58 | } 59 | 60 | function merge(orig, custom) { 61 | for (key in custom) { 62 | orig[key] = custom[key]; 63 | } 64 | } 65 | 66 | function load(inputFile, configFile) { 67 | if (configFile) { 68 | // Attempt to load config from file, exit if fails 69 | try { 70 | var data = fs.readFileSync(configFile, 'utf8'); 71 | loadedConfig = JSON.parse(data); 72 | } catch(e) { 73 | console.log("Error: invalid data found while trying to read config file. Please make sure it exists and contains valid JSON data."); 74 | process.exit(1); 75 | } 76 | merge(defaultConfig, loadedConfig); 77 | } 78 | defaultConfig.inputFile = inputFile; 79 | } 80 | 81 | var Config = {}; 82 | Config.load = load; 83 | Config.getRepresentationKeyForFile = getRepresentationKeyForFile; 84 | Object.defineProperty(Config, 'inputFile', { get: function() { return defaultConfig.inputFile; }}); 85 | Object.defineProperty(Config, 'segmentDir', { get: function() { return defaultConfig.segmentDir; }}); 86 | Object.defineProperty(Config, 'outputDir', { get: function() { return defaultConfig.outputDir; }}); 87 | Object.defineProperty(Config, 'ffmpeg', { get: function() { return defaultConfig.ffmpeg; }}); 88 | Object.defineProperty(Config, 'mp4box', { get: function() { return defaultConfig.mp4box; }}); 89 | Object.defineProperty(Config, 'encoding', { get: function() { return defaultConfig.encoding; }}); 90 | Object.defineProperty(Config, 'packaging', { get: function() { return defaultConfig.packaging; }}); 91 | Object.defineProperty(Config, 'representationKeys', { get: function() { return getRepresentationKeys(); }}); 92 | Object.defineProperty(Config, 'representationCount', { get: function() { return getRepresentationKeys().length; }}); 93 | 94 | module.exports = Config; -------------------------------------------------------------------------------- /lib/live-stream-controller.js: -------------------------------------------------------------------------------- 1 | var config = require('./config'), 2 | mkdirp = require('mkdirp'), 3 | watcher = require('./watcher'), 4 | transcoder = require('./transcoder'), 5 | reader = require('./reader'); 6 | 7 | function launch(inputFile, configFile) { 8 | config.load(inputFile, configFile); 9 | 10 | // Create input/output directories 11 | mkdirp.sync(config.segmentDir); 12 | mkdirp.sync(config.outputDir); 13 | 14 | // Initiate watcher 15 | watcher.watchSegmentDir(); 16 | 17 | if (inputFile.match(/^.+\:\/\//)) { 18 | // Protocol specified 19 | console.log('Protocol found for input file: ' + inputFile + '. Passing directly to ffmpeg.'); 20 | transcoder.spawn(inputFile); 21 | } else { 22 | // Spawn transcoder using default operation (pipes) 23 | var t = transcoder.spawn(); 24 | // Pipe reader input to transcoder 25 | reader.pipeStream(t.stdin); 26 | } 27 | } 28 | 29 | module.exports.launch = launch; -------------------------------------------------------------------------------- /lib/packager.js: -------------------------------------------------------------------------------- 1 | var child_process = require('child_process'), 2 | path = require('path'), 3 | config = require('./config'), 4 | fs = require('fs'); 5 | 6 | var prevTimestamp = new Date(); 7 | var avgTimestamp = 0; 8 | var runs = 0 9 | 10 | function packageSegments(segments) { 11 | var newTimestamp = new Date() 12 | 13 | processDashContext(segments, function(err, results) { 14 | if (err) { 15 | console.log("Error while processing segments: " + err); 16 | process.exit(1); 17 | } 18 | 19 | console.log("Results in: " + results); 20 | if (runs == 0) { 21 | console.log("The packager has processed: " + segments + "."); 22 | } else { 23 | var bm = newTimestamp - prevTimestamp; 24 | avgTimestamp = (avgTimestamp * (1 - 1/runs)) + (bm / runs); 25 | console.log("The packager has processed: " + segments + ". Last run was " + bm + "ms ago. Average time between runs: " + avgTimestamp + "ms."); 26 | } 27 | 28 | // Unlink all processed segments to clean up 29 | segments.map(function(item) { return path.join(config.segmentDir, item) }).concat(results).forEach(function(item) { 30 | fs.unlink(item, function(err) { /* Don't care about errors for now */ }); 31 | }) 32 | 33 | runs++; 34 | prevTimestamp = newTimestamp; 35 | }); 36 | } 37 | 38 | function processSingle(segment, cb) { 39 | var input = path.join(config.segmentDir, segment); 40 | var output = path.join(config.outputDir, segment + ".mp4"); 41 | 42 | console.log("Packaging single segment: " + segment); 43 | 44 | var packager = child_process.spawn(config.mp4box, ['-add', input, output]); 45 | packager.on('error', function(err) { cb(err, null); }); 46 | packager.on('exit', function(code) { 47 | fs.unlink(input, function(err) { /* Don't care about errors for now */ }); 48 | code == 0 ? cb(null, output) : cb(code, null); 49 | }); 50 | } 51 | 52 | function processDashContext(segments, cb) { 53 | // add the correct representation ids 54 | var representationFiles = segments.sort().map(function(s) { return s + ":id=" + config.getRepresentationKeyForFile(s); }); 55 | var contextPackager = child_process.spawn(config.mp4box, config.packaging.mp4box_opts.concat(representationFiles), { stdio: 'inherit' }); 56 | contextPackager.on('error', function(err) { cb(err, null); }); 57 | contextPackager.on('exit', function(code) { code == 0 ? cb(null, segments) : cb(code, null); }); 58 | 59 | console.log("Creating/updating DASH context. " + new Date()); 60 | } 61 | 62 | module.exports.packageSegments = packageSegments; 63 | module.exports.processSingle = processSingle; -------------------------------------------------------------------------------- /lib/reader.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'), 2 | config = require('./config'); 3 | 4 | function pipeStream(dst) { 5 | var readStream = fs.createReadStream(config.inputFile); 6 | 7 | readStream.on('open', function () { 8 | readStream.pipe(dst, { end: false }); 9 | }); 10 | 11 | readStream.on('end', function() { 12 | pipeStream(dst); 13 | }) 14 | 15 | readStream.on('error', function() { 16 | console.log("Unable to open input file for reading."); 17 | process.exit(1); 18 | }) 19 | } 20 | 21 | module.exports.pipeStream = pipeStream; 22 | -------------------------------------------------------------------------------- /lib/transcoder.js: -------------------------------------------------------------------------------- 1 | var config = require('./config'), 2 | child_process = require('child_process'), 3 | path = require('path'); 4 | 5 | function spawn(directInput) { 6 | return child_process.spawn(config.ffmpeg, buildOpts(directInput), { stdio: ['pipe', process.stdout, process.stderr] }); 7 | } 8 | 9 | function buildOpts(directInput) { 10 | var opts = config.encoding.commandPrefix; 11 | if (directInput) { 12 | var idx = opts.indexOf('-'); 13 | if (idx != -1) opts[idx] = directInput; 14 | } 15 | config.representationKeys.forEach(function(key) { 16 | opts = opts.concat(config.encoding.representations[key]).concat(path.join(config.segmentDir, key + "_%d.ts")); 17 | }); 18 | return opts; 19 | } 20 | 21 | module.exports.spawn = spawn; -------------------------------------------------------------------------------- /lib/watcher.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'), 2 | config = require('./config'), 3 | packager = require('./packager'); 4 | 5 | var availableSegments = []; 6 | var packagedSegments = []; 7 | var completedSegments = []; 8 | 9 | function watchSegmentDir() { 10 | var watcher = fs.watch(config.segmentDir); 11 | 12 | watcher.on('change', function(event, filename) { 13 | updateSegments(filename); 14 | }) 15 | 16 | watcher.on('error', function(err) { 17 | console.log("Got error " + err); 18 | }) 19 | } 20 | 21 | function updateSegments(filename) { 22 | // If the filename is a .ts file and not already processed 23 | if (/\.ts$/.test(filename) && 24 | availableSegments.indexOf(filename) == -1 && 25 | packagedSegments.indexOf(filename) == -1 && 26 | completedSegments.indexOf(filename) == -1) { 27 | availableSegments.push(filename); 28 | console.log("Available segments for DASHing: " + availableSegments); 29 | } 30 | 31 | // process a new single segment as soon as it's available 32 | if (availableSegments.length > config.representationCount) { 33 | // Get current representation 34 | var representationKey = config.getRepresentationKeyForFile(filename); 35 | 36 | // Get current index 37 | var newIndex = parseInt(filename.split('.')[0].split('_').pop(), 10); 38 | 39 | var oldKey = representationKey + "_" + (newIndex - 1) + ".ts"; 40 | var index = availableSegments.indexOf(oldKey); 41 | 42 | if (index > -1) { 43 | var segment = availableSegments.splice(index, 1)[0]; 44 | packager.processSingle(segment, function(err, result) { 45 | // See if we need to process the MPD 46 | if (result) { 47 | packagedSegments.push(result); 48 | updateMPD(); 49 | } 50 | }); 51 | } 52 | } 53 | } 54 | 55 | function updateMPD() { 56 | if (packagedSegments.length % config.representationCount == 0 && packagedSegments.length > 0) { 57 | var segmentsToBeProcessed = packagedSegments.splice(0, config.representationCount); 58 | // only keep 100 segments in memory to prevent unbounded growth 59 | completedSegments = segmentsToBeProcessed.concat(completedSegments).slice(0,100); 60 | console.log("Processing segments: " + segmentsToBeProcessed); 61 | packager.packageSegments(segmentsToBeProcessed); 62 | } 63 | } 64 | 65 | module.exports.watchSegmentDir = watchSegmentDir; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test-engine-live-tools", 3 | "description": "NodeJS tools to live stream an MPEG-TS using MPEG-DASH", 4 | "version": "1.0.1", 5 | "keywords": ["transcoding", "ffmpeg", "video", "mpeg", "dash", "MP4Box"], 6 | "repository": { 7 | "type": "git", 8 | "url": "https://github.com/ebu/test-engine-live-tools" 9 | }, 10 | "bin": { "live-stream": "./bin/live-stream" }, 11 | "engines": { 12 | "node": ">=0.10.25" 13 | }, 14 | "dependencies": { 15 | "mkdirp": "~0.5.0", 16 | "minimist": "~1.1.0" 17 | }, 18 | "licenses": [{ 19 | "type": "BSD-3-Clause", 20 | "url": "https://github.com/ebu/test-engine-live-tools/blob/master/LICENSE" 21 | }] 22 | } --------------------------------------------------------------------------------