├── .gitignore ├── Analytics.js ├── Commands.js ├── Console.js ├── Db.js ├── Display.js ├── Formaters.js ├── Gruntfile.js ├── README.md └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | config.db 2 | node_modules/ 3 | -------------------------------------------------------------------------------- /Analytics.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | var interval = 1000; 3 | var Display = require('./Display'); 4 | 5 | 6 | 7 | var _downloaded = function() { 8 | return _.reduce(this.threads, function(bytes, thread) { 9 | return bytes + thread.position - thread.start; 10 | }, 0); 11 | }; 12 | 13 | //Should be called on start 14 | var _past_downloaded = function() { 15 | this.past.downloaded = _downloaded.call(this); 16 | }; 17 | 18 | var _total_size = function() { 19 | this.total.size = _.last(this.threads).end - _.first(this.threads).start; 20 | }; 21 | 22 | //Should be called everytime 23 | var _present_time = function() { 24 | this.present.time += (interval / 1000); 25 | }; 26 | 27 | var _total_downloaded = function() { 28 | this.total.downloaded = _downloaded.call(this); 29 | }; 30 | 31 | var _present_downloaded = function() { 32 | this.present.downloaded = this.total.downloaded - this.past.downloaded; 33 | }; 34 | 35 | var _total_completed = function() { 36 | this.total.completed = Math.floor((this.total.downloaded) * 1000 / this.total.size) / 10; 37 | }; 38 | 39 | 40 | var _future_remaining = function() { 41 | this.future.remaining = this.total.size - this.total.downloaded; 42 | }; 43 | 44 | var _present_speed = function() { 45 | this.present.speed = this.present.downloaded / this.present.time; 46 | }; 47 | 48 | var _future_eta = function() { 49 | this.future.eta = this.future.remaining / this.present.speed; 50 | }; 51 | 52 | var _present_threadStatus = function() { 53 | this.present.threadStatus = _.reduce(this.threads, function(memo, thread) { 54 | memo[thread.connection]++; 55 | return memo; 56 | }, { 57 | idle: 0, 58 | open: 0, 59 | closed: 0, 60 | failed: 0 61 | 62 | }); 63 | }; 64 | 65 | 66 | var Analytics = function() { 67 | this.past = { 68 | downloaded: 0 69 | }; 70 | 71 | this.present = { 72 | downloaded: 0, 73 | time: 0 74 | }; 75 | 76 | this.future = {}; 77 | this.total = {}; 78 | this.elapsed = 0; 79 | }; 80 | 81 | var _init = function(threads) { 82 | this.threads = threads; 83 | _past_downloaded.call(this); 84 | _total_size.call(this); 85 | }; 86 | 87 | var _update = function() { 88 | _present_time.call(this); 89 | _total_downloaded.call(this); 90 | _present_downloaded.call(this); 91 | _total_completed.call(this); 92 | _future_remaining.call(this); 93 | _present_speed.call(this); 94 | _future_eta.call(this); 95 | _present_threadStatus.call(this); 96 | }; 97 | 98 | var _start = function(threads) { 99 | var self = this; 100 | 101 | _init.call(self, threads); 102 | Display.newLine(); 103 | self.timer = setInterval(function() { 104 | _update.call(self); 105 | Display.show(self); 106 | }, interval); 107 | 108 | Display.header(); 109 | }; 110 | 111 | var _stop = function() { 112 | if (this.timer) { 113 | clearInterval(this.timer); 114 | _update.call(this); 115 | Display.show(this); 116 | Display.newLine(); 117 | Display.newLine(); 118 | } 119 | }; 120 | 121 | Analytics.prototype.start = _start; 122 | Analytics.prototype.stop = _stop; 123 | module.exports = Analytics; -------------------------------------------------------------------------------- /Commands.js: -------------------------------------------------------------------------------- 1 | //REQUIRES 2 | var minimist = require('minimist'); 3 | var Analytics = require('./Analytics'); 4 | var mtd = require('mt-downloader'); 5 | var f = require('./Formaters'); 6 | var Package = require('./package.json'); 7 | var _ = require('underscore'); 8 | var Db = require('./Db'); 9 | 10 | var log = console.log; 11 | 12 | //OBJECTS 13 | var analytics = new Analytics(); 14 | var store = new Db('config.db'); 15 | 16 | var Commands = function () {}; 17 | 18 | var _auto_name = function (url, callback) { 19 | 20 | var _name = function (url) { 21 | return _.reduce(['&', '/', '='], function (memo, item) { 22 | var _url = decodeURI(memo).split(item); 23 | return _url[_url.length - 1]; 24 | }, url); 25 | }; 26 | 27 | var name = _name(url); 28 | if (name.length < 1) { 29 | name = (new Date()).toISOString(); 30 | } 31 | store.get('wd', function (val) { 32 | callback(val + name); 33 | }); 34 | 35 | }; 36 | 37 | var _show_help = function () { 38 | console.log('To know more visit [https://github.com/tusharmath/mtd-console]'); 39 | }; 40 | 41 | var _set_wd = function (args) { 42 | store.save('wd', args['set-wd'], function () { 43 | console.log('Working directory updated:', args['set-wd']); 44 | }); 45 | }; 46 | 47 | var _clear_wd = function () { 48 | store.remove('wd', function () { 49 | console.log('Working directory cleared.'); 50 | }); 51 | }; 52 | 53 | var _wd = function () { 54 | store.get('wd', function (value) { 55 | if (value) console.log('Working directory:', value); 56 | console.log('Set working directory using [--set-wd]'); 57 | }); 58 | }; 59 | 60 | var _start_download = function (args) { 61 | 62 | args.onStart = function (data) { 63 | 64 | console.log('Size:', f.byteFormater(data.size)); 65 | console.log('Url:', data.url); 66 | if (data.headers && data.headers['content-disposition'] && data.headers['content-disposition'].indexOf('filename=') > -1) console.log('Attachment name:', data.headers['content-disposition'].split('filename=')[1].replace(/[\"]/g, '')); 67 | analytics.start(data.threads); 68 | 69 | }; 70 | 71 | args.onEnd = function (err, data) { 72 | analytics.stop(); 73 | if (err) console.error(err); 74 | else console.log('Downloaded'); 75 | }; 76 | if (args.headers && args.headers.length > 0) { 77 | var _headers = _.reduce(args.headers.split(';'), function (header, val) { 78 | var ref = val.split(':'), 79 | headerName = ref[0], 80 | headerValue = ref[1]; 81 | header[headerName] = headerValue; 82 | return header; 83 | }, {}); 84 | args.headers = _headers; 85 | } 86 | 87 | if (args['url'] !== undefined && args['file'] === undefined) { 88 | _auto_name(args.url, function (file) { 89 | args.file = file; 90 | var downloader = new mtd(file, args.url, args); 91 | downloader.start(); 92 | console.log('File:', args.file); 93 | }); 94 | } else store.get('wd', function (value) { 95 | args.file = value + args.file; 96 | var downloader = new mtd(args.file, args.url, args); 97 | downloader.start(); 98 | console.log('File:', args.file); 99 | }); 100 | 101 | }; 102 | 103 | var _show_version = function () { 104 | log('Version:', Package.version); 105 | }; 106 | 107 | var _get_action = function (args) { 108 | if (args['url'] || args['file']) return _start_download; 109 | if (args['set-wd']) return _set_wd; 110 | if (args['clear-wd']) return _clear_wd; 111 | if (args['wd']) return _wd; 112 | if (args['version']) return _show_version; 113 | if (args['help']) return _show_help; 114 | return _show_help; 115 | }; 116 | 117 | Commands.prototype.execute = function (argv) { 118 | this.args = minimist(argv.slice(2)); 119 | var action = _get_action(this.args); 120 | action(this.args); 121 | }; 122 | 123 | module.exports = Commands; 124 | -------------------------------------------------------------------------------- /Console.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var _ = require('underscore'); 4 | var Package = require('./Package.json'); 5 | var Commands = require('./Commands'); 6 | var cmd = new Commands(); 7 | var title = '\nmt-Console (version: ' + Package.version + ')'; 8 | console.log(title); 9 | _.times(title.length - 1, function() { 10 | process.stdout.write('='); 11 | }); 12 | console.log('\n'); 13 | 14 | cmd.execute(process.argv); -------------------------------------------------------------------------------- /Db.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'); 2 | 3 | var Db = function(file) { 4 | this.file = __dirname + file; 5 | }; 6 | 7 | var readFile = function(callback) { 8 | fs.readFile(this.file, { 9 | encoding: 'utf8' 10 | }, function(err, data) { 11 | if (data) { 12 | callback(JSON.parse(data)); 13 | } else { 14 | callback({}); 15 | } 16 | }); 17 | }; 18 | 19 | var writeFile = function(content, callback) { 20 | fs.writeFile(this.file, JSON.stringify(content), { 21 | encoding: 'utf8' 22 | }, callback); 23 | }; 24 | 25 | Db.prototype.get = function(key, callback) { 26 | readFile.call(this, function(content) { 27 | callback(content[key] || ''); 28 | }); 29 | }; 30 | 31 | Db.prototype.save = function(key, value, callback) { 32 | var self = this; 33 | readFile.call(this, function(content) { 34 | content[key] = value; 35 | writeFile.call(self, content, callback); 36 | }); 37 | }; 38 | 39 | Db.prototype.remove = function(key, callback) { 40 | var self = this; 41 | readFile.call(this, function(content) { 42 | delete content[key]; 43 | writeFile.call(self, content, callback); 44 | }); 45 | }; 46 | 47 | module.exports = Db; -------------------------------------------------------------------------------- /Display.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | var Formater = require('./Formaters'); 3 | 4 | var Display = {}; 5 | 6 | var _logInline = function(str) { 7 | //return; 8 | process.stdout.clearLine(); // clear current text 9 | process.stdout.cursorTo(0); // move cursor to beginning of line 10 | process.stdout.write(str.toString()); // write text 11 | }; 12 | 13 | var _display = function(analytics) { 14 | var str = Formater.rightPad([ 15 | analytics.total.completed + '%', 16 | Formater.speedFormater(analytics.present.speed), 17 | Formater.elapsedTimeFormater(analytics.present.time), 18 | Formater.remainingTimeFormater(analytics.future.eta), 19 | Formater.threadStatusFormater(analytics.present.threadStatus)]); 20 | _logInline(str); 21 | }; 22 | 23 | Display.header = function() { 24 | console.log(Formater.rightPad(['Completed', 'Speed', 'Time', 'ETA', 'Status (I, O, C, F)'])); 25 | }; 26 | 27 | Display.show = _display; 28 | Display.inline = _logInline; 29 | Display.newLine = function() { 30 | console.log(); 31 | }; 32 | 33 | 34 | module.exports = Display; -------------------------------------------------------------------------------- /Formaters.js: -------------------------------------------------------------------------------- 1 | var moment = require('moment'); 2 | var _ = require('underscore'); 3 | var col = 20; 4 | 5 | var _floor = function(val) { 6 | return Math.floor(val); 7 | }; 8 | 9 | var _rightPad = function(memo, str) { 10 | return memo + str + (new Array(col - str.length)).join(' '); 11 | }; 12 | 13 | exports.elapsedTimeFormater = function(seconds) { 14 | return _floor(seconds) + 's'; 15 | }; 16 | 17 | 18 | exports.remainingTimeFormater = function(seconds) { 19 | return moment.duration(seconds, 'seconds').humanize(); 20 | }; 21 | 22 | exports.byteFormater = function(bytes) { 23 | var str; 24 | if (bytes > 1024 * 1024 * 1024) str = _floor(bytes * 100 / (1024 * 1024 * 1024)) / 100 + ' GB'; 25 | else if (bytes > 1024 * 1024) str = _floor(bytes * 100 / (1024 * 1024)) / 100 + ' MB'; 26 | else if (bytes > 1024) str = _floor(bytes * 100 / 1024) / 100 + ' KB'; 27 | else str = _floor(bytes) + ' Bytes'; 28 | return str; 29 | 30 | }; 31 | 32 | exports.speedFormater = function(speed) { 33 | var str; 34 | speed *= 8; 35 | if (speed > 1024 * 1024) str = _floor(speed * 10 / (1024 * 1024)) / 10 + ' Mbps'; 36 | else if (speed > 1024) str = _floor(speed * 10 / 1024) / 10 + ' Kbps'; 37 | else str = _floor(speed) + ' bps'; 38 | return str + ' '; 39 | }; 40 | 41 | exports.rightPad = function(list) { 42 | return _.reduce(list, _rightPad, ''); 43 | }; 44 | 45 | exports.threadStatusFormater = function(status) { 46 | return [status.idle, status.open, status.closed, status.failed].join(', '); 47 | }; -------------------------------------------------------------------------------- /Gruntfile.js: -------------------------------------------------------------------------------- 1 | var _config = { 2 | release: { 3 | options: { 4 | //bump: false, //default: true 5 | //file: 'component.json', //default: package.json 6 | //add: false, //default: true 7 | //commit: false, //default: true 8 | //tag: false, //default: true 9 | //push: false, //default: true 10 | //pushTags: false, //default: true 11 | //npm: false, //default: true 12 | //tagName: 'some-tag-<%= version %>', //default: '<%= version %>' 13 | //commitMessage: 'New release <%= version %>', //default: 'release <%= version %>' 14 | //tagMessage: 'tagging version <%= version %>' //default: 'Version <%= version %>' 15 | } 16 | } 17 | }; 18 | 19 | module.exports = function(grunt) { 20 | grunt.initConfig(_config); 21 | grunt.loadNpmTasks('grunt-release'); 22 | }; -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #mt-console 2 | 3 | A platform independent console application for [mt-downloader](https://github.com/tusharmath/Multi-threaded-downloader). This app is only an abstraction of what the original library can do. To get a complete list of features you should see [mt-downloader](https://github.com/tusharmath/Multi-threaded-downloader). 4 | 5 | ##Installation 6 | 7 | Install this globally using the conventional npm installation command. 8 | 9 | ```bash 10 | $ npm install -g mt-console 11 | ``` 12 | 13 | ##Usage 14 | 15 | 1. You can get started by using the ```--help``` option. This lists out all the possible operations in mtd. 16 | 17 | ```bash 18 | $ mtd --help 19 | ``` 20 | 21 | 2. To start a new download you will have to provide a ```--url``` and a download ```--file``` path. 22 | 23 | ```bash 24 | $ mtd --url="http://path/to/file.zip" --file="/Downloads/file.zip" 25 | ``` 26 | 27 | **NOTE:** Make sure you use the double quotes because some times if you have spaces in your paths it creates problems. 28 | 29 | 3. To resume an old download, you just need to provide the path to the file with .mtd extension that is temporarily created at the time of download. 30 | 31 | ```bash 32 | $ mtd --file="/Downloads/file.zip.mtd" 33 | ``` 34 | 35 | 4. You can also pass custom options such as - 36 | 37 | a. ```--count``` : To set a custom number of download threads. It defaults to what is set in the [mt-downloader](https://github.com/tusharmath/Multi-threaded-downloader) library. 38 | 39 | b. ```--range``` : You can specify a custom download range. This feature is particularly useful when you want to download a part of a video file. Say you just want to download the later half of the file, you can then set the range as **50-100**. Its an optional parameter and defaults to **0-100**. 40 | 41 | c. ```--port``` : You can specify a custom HTTP port. It defaults to **80**. 42 | 43 | d. ```--method``` : You can specify the download method such as **PUT** and **POST** by default it is set to **GET**. 44 | 45 | e. ```--wd``` : Shows the current working directory. You can update this using ```--set-wd``` option. 46 | 47 | f. ```--set-wd``` : You can set you current working directory using this command. This is particularly helpful if you want to avoid typing the complete download path with the ```--file``` parameter. Instead you can set a common path for downloads and just specify the name of the file. The app will automatically combine the two and create the complete file path. 48 | 49 | ```bash 50 | $ mtd --set-wd="/Users/tusharmathur/Downloads/" 51 | $ Working directory updated to /Users/tusharmathur/Downloads/ 52 | 53 | $ mtd --wd 54 | Working directory: /Users/tusharmathur/Downloads/ 55 | 56 | $ mtd --url="http://path/to/file_one.zip" --file="file_one.zip" 57 | $ mtd --url="http://path/to/file_two.zip" --file="file_two.zip" 58 | 59 | ``` 60 | 61 | Both the files *file_one.zip* and *file_two.zip* will be downloaded at the same location ```/Users/tusharmathur/Downloads/``` because that is the default download path. 62 | 63 | g. ```--clear-wd``` : Clears the saved working directory. 64 | 65 | h. ```--timeout``` : Sometimes the connections are established but are not transferring any data. Using this setting you can set the maximum amount of time in **seconds** that it should wait before quitting. 66 | 67 | i. ```--auto-name``` : Generates the file name on its own by parsing the last element of the url. You will not be required to set the ```--file``` parameter while starting a new download. It will also automatically prepend the generated file name with the working directory specified by the ```--set-wd``` option. 68 | 69 | j. ```--headers``` : You can specify headers by seperating them using semicolons as follows 70 | 71 | ```bash 72 | $ mtd --url="http://path/to/file_one.zip" --file="file_one.zip" --headers="user-agent:crawl-bot;cookie:abc%3D100%3Bpqr%3D200" 73 | ``` 74 | 75 | **NOTE:** make sure you encode the header values parameters while sending else they might get escaped while parsing. 76 | 77 | If you want to know more about this app you can visit [tusharm.com](http://tusharm.com/articles/mt-downloader). Hope this helps you in downloading your data more efficiently! 78 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mt-console", 3 | "description": "A Console application for mt-downloader", 4 | "version": "0.0.11", 5 | "author": { 6 | "name": "Tushar Mathur ", 7 | "url": "http://tusharm.com" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/tusharmath/mtd-console" 12 | }, 13 | "engines": { 14 | "node": ">= 0.10.0" 15 | }, 16 | "bin": { 17 | "mtd": "./Console.js" 18 | }, 19 | "preferGlobal": true, 20 | "dependencies": { 21 | "underscore": "1.5.x", 22 | "minimist": "0.0.x", 23 | "mt-downloader": "0.2.x", 24 | "moment": "2.0.x" 25 | }, 26 | "devDependencies": { 27 | "grunt-release": "0.3.x" 28 | } 29 | } 30 | --------------------------------------------------------------------------------