├── .gitignore ├── LICENSE ├── README.md ├── bin └── nwget.js ├── index.js ├── lib └── wget.js ├── package.json └── test └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | (The MIT License) 2 | 3 | Copyright (c) 2012 Chengwei Wu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # node-wget 2 | 3 | A download tool, now supporting http/https resource and http/https proxy, written in nodejs. 4 | 5 | # Installing 6 | ``` 7 | npm install wget 8 | ``` 9 | 10 | # Usage 11 | 12 | 13 | ## download(src, output, options) 14 | 15 | ```js 16 | var wget = require('wget'); 17 | var src = 'https://raw.github.com/Fyrd/caniuse/master/data.json'; 18 | var output = '/tmp/data.json'; 19 | var options = { 20 | proxy: 'http://host:port' 21 | }; 22 | var download = wget.download(src, output, options); 23 | download.on('error', function(err) { 24 | console.log(err); 25 | }); 26 | download.on('end', function(output) { 27 | console.log(output); 28 | }); 29 | download.on('progress', function(progress) { 30 | // code to show progress bar 31 | }); 32 | ``` 33 | 34 | 35 | ## request(options, callback) 36 | 37 | ```js 38 | var wget = require('wget'); 39 | var options = { 40 | protocol: 'https', 41 | host: 'raw.github.com', 42 | path: '/Fyrd/caniuse/master/data.json', 43 | proxy: 'http://host:port', 44 | method: 'GET' 45 | }; 46 | var req = wget.request(options, function(res) { 47 | var content = ''; 48 | if (res.statusCode === 200) { 49 | res.on('error', function(err) { 50 | console.log(err); 51 | }); 52 | res.on('data', function(chunk) { 53 | content += chunk; 54 | }); 55 | res.on('end', function() { 56 | console.log(content); 57 | }); 58 | } else { 59 | console.log('Server respond ' + res.statusCode); 60 | } 61 | }); 62 | 63 | req.end(); 64 | req.on('error', function(err) { 65 | console.log(err); 66 | }); 67 | ``` -------------------------------------------------------------------------------- /bin/nwget.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wuchengwei/node-wget/ee0cbec1a928d5d429dcc85c973cdd28d8b23288/bin/nwget.js -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | exports.download = require('./lib/wget').download; 2 | exports.request = require('./lib/wget').request; -------------------------------------------------------------------------------- /lib/wget.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var http = require('http'); 4 | var https = require('https'); 5 | var tunnel = require('tunnel'); 6 | var url = require('url'); 7 | var util = require('util'); 8 | var fs = require('fs'); 9 | var EventEmitter = require('events').EventEmitter; 10 | 11 | function download(src, output, options) { 12 | var downloader = new EventEmitter(), 13 | srcUrl, 14 | tunnelAgent, 15 | req; 16 | 17 | if (options) { 18 | options = parseOptions('download', options); 19 | } 20 | srcUrl = url.parse(src); 21 | srcUrl.protocol = cleanProtocol(srcUrl.protocol); 22 | 23 | req = request({ 24 | protocol: srcUrl.protocol, 25 | host: srcUrl.hostname, 26 | port: srcUrl.port, 27 | path: srcUrl.pathname, 28 | proxy: options?options.proxy:undefined, 29 | method: 'GET' 30 | }, function(res) { 31 | var fileSize, writeStream, downloadedSize; 32 | if (res.statusCode === 200) { 33 | downloadedSize = 0; 34 | fileSize = res.headers['content-length']; 35 | writeStream = fs.createWriteStream(output, { 36 | flags: 'a', 37 | encoding: 'binary' 38 | }); 39 | 40 | res.on('error', function(err) { 41 | writeStream.end(); 42 | downloader.emit('error', err); 43 | }); 44 | res.on('data', function(chunk) { 45 | downloadedSize += chunk.length; 46 | downloader.emit('progress', downloadedSize/fileSize); 47 | writeStream.write(chunk); 48 | }); 49 | res.on('end', function() { 50 | writeStream.end(); 51 | }); 52 | writeStream.on('close', function(){ 53 | downloader.emit('end', output); 54 | }); 55 | } else { 56 | downloader.emit('error', 'Server respond ' + res.statusCode); 57 | } 58 | }); 59 | 60 | req.end(); 61 | req.on('error', function(err) { 62 | downloader.emit('error', err); 63 | }); 64 | 65 | return downloader; 66 | } 67 | 68 | function request(options, callback) { 69 | options = parseOptions('request', options); 70 | if (options.protocol === 'http') { 71 | if (options.proxy) { 72 | if (options.proxy.protocol === 'http') { 73 | delete options.proxy.protocol; // delete self-defined arg 74 | options.agent = tunnel.httpOverHttp({proxy: options.proxy}); 75 | } else if (options.proxy.protocol === 'https') { 76 | delete options.proxy.protocol; // delete self-defined arg 77 | options.agent = tunnel.httpOverHttps({proxy: options.proxy}); 78 | } else { 79 | throw options.proxy.protocol + ' proxy is not supported!'; 80 | } 81 | } 82 | delete options.protocol; // delete self-defined arg 83 | delete options.proxy; // delete self-defined arg 84 | return http.request(options, callback); 85 | } 86 | if (options.protocol === 'https') { 87 | if (options.proxy) { 88 | if (options.proxy.protocol === 'http') { 89 | delete options.proxy.protocol; // delete self-defined arg 90 | options.agent = tunnel.httpsOverHttp({proxy: options.proxy}); 91 | } else if (options.proxy.protocol === 'https') { 92 | delete options.proxy.protocol; // delete self-defined arg 93 | options.agent = tunnel.httpsOverHttps({proxy: options.proxy}); 94 | } else { 95 | throw options.proxy.protocol + ' proxy is not supported!'; 96 | } 97 | } 98 | delete options.protocol; // delete self-defined arg 99 | delete options.proxy; // delete self-defined arg 100 | return https.request(options, callback); 101 | } 102 | throw 'only allow http or https request!'; 103 | } 104 | 105 | function parseOptions(type, options) { 106 | var proxy; 107 | if (type === 'download') { 108 | if (options.proxy) { 109 | if (typeof options.proxy === 'string') { 110 | proxy = url.parse(options.proxy); 111 | options.proxy = {}; 112 | options.proxy.protocol = cleanProtocol(proxy.protocol); 113 | options.proxy.host = proxy.hostname; 114 | options.proxy.port = proxy.port; 115 | options.proxy.proxyAuth = proxy.auth; 116 | options.proxy.headers = {'User-Agent': 'Node'}; 117 | } 118 | } 119 | return options; 120 | } 121 | if (type === 'request') { 122 | if (!options.protocol) { 123 | options.protocol = 'http'; 124 | } 125 | options.protocol = cleanProtocol(options.protocol); 126 | 127 | if (options.proxy) { 128 | if (typeof options.proxy === 'string') { 129 | proxy = url.parse(options.proxy); 130 | options.proxy = {}; 131 | options.proxy.protocol = cleanProtocol(proxy.protocol); 132 | options.proxy.host = proxy.hostname; 133 | options.proxy.port = proxy.port; 134 | options.proxy.proxyAuth = proxy.auth; 135 | options.proxy.headers = {'User-Agent': 'Node'}; 136 | } 137 | } 138 | return options; 139 | } 140 | } 141 | 142 | function cleanProtocol(str) { 143 | return str.trim().toLowerCase().replace(/:$/, ''); 144 | } 145 | 146 | exports.download = download; 147 | exports.request = request; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "wget", 3 | "version": "0.0.1", 4 | "description": "wget in nodejs.", 5 | "keywords": ["download", "http", "https", "ftp", "proxy"], 6 | "author": "Chengwei Wu ", 7 | "repository":{ 8 | "type": "git", 9 | "url": "git://github.com/wuchengwei/node-wget.git" 10 | }, 11 | "main": "./index.js", 12 | "bin": { 13 | "nwget": "./bin/nwget.js" 14 | }, 15 | "dependencies": { 16 | "tunnel": "0.0.2" 17 | }, 18 | "engines": { "node": ">= 0.6.18" } 19 | } 20 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | var wget = require('../lib/wget'); 2 | 3 | var download = wget.download('https://raw.github.com/Fyrd/caniuse/master/data.json', '/tmp/README.md'); 4 | // with a proxy: 5 | // var download = wget.download('https://raw.github.com/Fyrd/caniuse/master/data.json', '/tmp/README.md', {proxy: 'http://proxyhost:port'}); 6 | download.on('error', function(err) { 7 | console.log(err); 8 | }); 9 | download.on('end', function(output) { 10 | console.log(output); 11 | }); 12 | download.on('progress', function(progress) { 13 | console.log(progress); 14 | }); --------------------------------------------------------------------------------