├── .editorconfig ├── .gitignore ├── .npmignore ├── .travis.yml ├── History.md ├── Makefile ├── README.md ├── index.js ├── package.json ├── request.js └── test └── index.test.js /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | 15 | [makefile] 16 | indent_style = tab 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.seed 2 | *.log 3 | *.csv 4 | *.dat 5 | *.out 6 | *.pid 7 | *.gz 8 | 9 | coverage.html 10 | coverage/ 11 | cov/ 12 | 13 | node_modules 14 | 15 | dump.rdb 16 | .DS_Store 17 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | benchmark/ 2 | test/ 3 | cov/ 4 | Makefile 5 | covrage.html 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | sudo: false 3 | node_js: 4 | - "4" 5 | - "5" 6 | - "6" 7 | script: "make test-travis" 8 | after_script: "npm install coveralls@2.10.0 && cat ./coverage/lcov.info | coveralls" 9 | -------------------------------------------------------------------------------- /History.md: -------------------------------------------------------------------------------- 1 | 2 | 1.1.4 / 2017-05-19 3 | ================== 4 | 5 | * fix: package.json to reduce vulnerabilities (#6) 6 | 7 | 1.1.3 / 2016-05-03 8 | ================== 9 | 10 | * Update dependencies & travis (#4) 11 | 12 | 1.1.2 / 2015-09-30 13 | ================== 14 | 15 | * deps: urllib@2.5.0 16 | 17 | 1.1.1 / 2015-09-01 18 | ================== 19 | 20 | * fix: writeFile only succee with 2xx 21 | * deps: urllib@2.4.0 22 | 23 | 1.1.0 / 2015-05-05 24 | ================== 25 | 26 | * urllib@2 27 | * feat: support writeFile 28 | 29 | 1.0.1 / 2014-10-10 30 | ================== 31 | 32 | * add iconv-lite 33 | * update readme 34 | * add keywords 35 | * update readme 36 | * init 37 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TESTS = test/*.test.js 2 | REPORTER = tap 3 | TIMEOUT = 3000 4 | MOCHA_OPTS = 5 | 6 | install: 7 | @npm install --registry=https://registry.npmmirror.com 8 | 9 | test: 10 | @NODE_ENV=test ./node_modules/mocha/bin/mocha \ 11 | --reporter $(REPORTER) \ 12 | --timeout $(TIMEOUT) \ 13 | --require should \ 14 | $(MOCHA_OPTS) \ 15 | $(TESTS) 16 | 17 | test-cov: 18 | @NODE_ENV=test node \ 19 | node_modules/.bin/istanbul cover \ 20 | ./node_modules/.bin/_mocha \ 21 | -- -u exports \ 22 | --require should \ 23 | $(TESTS) \ 24 | --bail 25 | 26 | test-travis: 27 | @NODE_ENV=test node \ 28 | node_modules/.bin/istanbul cover \ 29 | ./node_modules/.bin/_mocha \ 30 | --report lcovonly \ 31 | -- -u exports \ 32 | --require should \ 33 | $(TESTS) \ 34 | --bail 35 | 36 | autod: install 37 | @node_modules/.bin/autod -w -e example.js --prefix=~ -D mocha,should,istanbul 38 | @$(MAKE) install 39 | 40 | .PHONY: test 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | urllib-sync 2 | --------------- 3 | 4 | [![NPM version][npm-image]][npm-url] 5 | [![build status][travis-image]][travis-url] 6 | [![Test coverage][coveralls-image]][coveralls-url] 7 | [![David deps][david-image]][david-url] 8 | [![node version][node-image]][node-url] 9 | [![Gittip][gittip-image]][gittip-url] 10 | 11 | [npm-image]: https://img.shields.io/npm/v/urllib-sync.svg?style=flat-square 12 | [npm-url]: https://npmjs.org/package/urllib-sync 13 | [travis-image]: https://img.shields.io/travis/node-modules/urllib-sync.svg?style=flat-square 14 | [travis-url]: https://travis-ci.org/node-modules/urllib-sync 15 | [coveralls-image]: https://img.shields.io/coveralls/node-modules/urllib-sync.svg?style=flat-square 16 | [coveralls-url]: https://coveralls.io/r/node-modules/urllib-sync?branch=master 17 | [david-image]: https://img.shields.io/david/node-modules/urllib-sync.svg?style=flat-square 18 | [david-url]: https://david-dm.org/node-modules/urllib-sync 19 | [node-image]: https://img.shields.io/badge/node.js-%3E=_0.11-red.svg?style=flat-square 20 | [node-url]: http://nodejs.org/download/ 21 | [gittip-image]: https://img.shields.io/gittip/dead-horse.svg?style=flat-square 22 | [gittip-url]: https://www.gittip.com/dead-horse/ 23 | 24 | sync http request powered by [urllib](https://github.com/node-modules/urllib) 25 | and spawnSync. 26 | 27 | ___Notice: Only support node v0.11.13+___ 28 | 29 | ## Installation 30 | 31 | ```bash 32 | $ npm install urllib-sync --save 33 | ``` 34 | 35 | ## Usage 36 | 37 | ```js 38 | var request = require('urllib-sync').request; 39 | 40 | var res = request('https://github.com'); 41 | // res should have status, data, headers 42 | ``` 43 | 44 | more options please check out [urllib](https://github.com/node-modules/urllib). But unfortunately, `urllib-sync` ___do not support streams and agents now___. 45 | 46 | ### writeFile 47 | 48 | If `options.writeFile` is present, the result of request will write into this file directly. 49 | 50 | 51 | ### License 52 | 53 | MIT 54 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * urllib-sync - index.js 3 | * Copyright(c) 2014 dead_horse 4 | * MIT Licensed 5 | */ 6 | 7 | 'use strict'; 8 | 9 | /** 10 | * Module dependencies. 11 | */ 12 | 13 | var cp = require('child_process'); 14 | var utility = require('utility'); 15 | var assert = require('assert'); 16 | var path = require('path'); 17 | var fs = require('fs'); 18 | 19 | assert(cp.spawnSync, 'urllib-sync need node version 0.11.13+'); 20 | 21 | exports.request = function request(url, args) { 22 | assert(url, 'url required'); 23 | args = args || {}; 24 | 25 | var input = utility.base64encode(JSON.stringify({ 26 | url: url, 27 | args: args 28 | })); 29 | var requestPath = path.join(__dirname, './request.js'); 30 | 31 | var cmd = process.execPath; 32 | var _args = [requestPath, input]; 33 | var res = cp.spawnSync(cmd, _args, { 34 | timeout: (args.timeout || 3000) + 1000 35 | }); 36 | 37 | if (res.error) { 38 | res.error.url = url; 39 | res.error.args = args; 40 | throw res.error; 41 | } 42 | 43 | if (res.status !== 0) { 44 | var e = new Error(res.stderr.toString() || 'unknown error'); 45 | e.url = url; 46 | e.args = args; 47 | e.status = res.status; 48 | throw e; 49 | } 50 | 51 | try { 52 | res = JSON.parse(res.stdout); 53 | } catch (err) { 54 | var e = new Error('parse response error:' + err.message); 55 | e.url = url; 56 | e.args = args; 57 | throw e; 58 | } 59 | 60 | try { 61 | switch (res.type) { 62 | case 'string': 63 | res.data = fs.readFileSync(res.path).toString(); 64 | break; 65 | case 'json': 66 | res.data = JSON.parse(fs.readFileSync(res.path).toString()); 67 | break; 68 | case 'file': 69 | res.data = null; 70 | break; 71 | default: 72 | res.data = fs.readFileSync(res.path); 73 | } 74 | } catch (err){ 75 | // ignore 76 | } finally { 77 | if (res.type !== 'file') fs.unlinkSync(res.path); 78 | delete res.path; 79 | } 80 | 81 | return res; 82 | }; 83 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "urllib-sync", 3 | "version": "1.1.4", 4 | "description": "sync http request", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "make test" 8 | }, 9 | "keywords": [ 10 | "http", 11 | "sync", 12 | "request" 13 | ], 14 | "author": { 15 | "name": "dead-horse", 16 | "email": "dead_horse@qq.com", 17 | "url": "http://deadhorse.me" 18 | }, 19 | "repository": { 20 | "type": "git", 21 | "url": "git@github.com:node-modules/urllib-sync" 22 | }, 23 | "license": "MIT", 24 | "dependencies": { 25 | "urllib": "~2.11.0", 26 | "utility": "~1.7.1" 27 | }, 28 | "devDependencies": { 29 | "autod": "2", 30 | "iconv-lite": "~0.4.13", 31 | "istanbul": "~0.4.3", 32 | "mocha": "~2.4.5", 33 | "should": "~8.3.1" 34 | }, 35 | "engine": { 36 | "node": ">=0.11.13" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /request.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * urllib-sync - request.js 3 | * Copyright(c) Alibaba Group Holding Limited. 4 | * Author: busi.hyy 5 | */ 6 | 7 | 'use strict'; 8 | 9 | /** 10 | * Module dependencies. 11 | */ 12 | 13 | var utility = require('utility'); 14 | var urllib = require('urllib'); 15 | var path = require('path'); 16 | var util = require('util'); 17 | var fs = require('fs'); 18 | var os = require('os'); 19 | 20 | var input = {}; 21 | try { 22 | input = utility.base64decode(process.argv[2] || ''); 23 | input = JSON.parse(input); 24 | } catch (err) { 25 | console.error(err.message); 26 | process.exit(1); 27 | } 28 | 29 | urllib.request(input.url, input.args, function (err, data, res) { 30 | if (err) { 31 | console.error(err.message); 32 | process.exit(1); 33 | } 34 | 35 | var name = util.format('%s:%s', process.pid, Date.now()); 36 | var type = 'buffer'; 37 | if (data && typeof data === 'object' && !Buffer.isBuffer(data)) { 38 | type = 'json'; 39 | data = JSON.stringify(data); 40 | } else if (typeof data === 'string') { 41 | type = 'string'; 42 | } 43 | 44 | var filepath = path.join(os.tmpDir(), name); 45 | 46 | // if need to writeFile 47 | if ((res.statusCode / 100 | 0) === 2 && input.args.writeFile) { 48 | type = 'file'; 49 | filepath = input.args.writeFile; 50 | } 51 | fs.writeFileSync(filepath, data); 52 | 53 | var res = { 54 | path: filepath, 55 | type: type, 56 | status: res.statusCode, 57 | headers: res.headers 58 | }; 59 | 60 | console.log(JSON.stringify(res)); 61 | process.exit(0); 62 | }); 63 | -------------------------------------------------------------------------------- /test/index.test.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * urllib-sync - test/index.test.js 3 | * Copyright(c) 2014 dead_horse 4 | * MIT Licensed 5 | */ 6 | 7 | 'use strict'; 8 | 9 | /** 10 | * Module dependencies. 11 | */ 12 | 13 | var urllib = require('..'); 14 | var iconv = require('iconv-lite'); 15 | var fs = require('fs'); 16 | 17 | describe('urllib-sync', function () { 18 | describe('request()', function () { 19 | it('should request gbk ok', function () { 20 | this.timeout(4000); 21 | var res = urllib.request('http://www.taobao.com/go/rgn/tmall/header/2014/sub-nav.php'); 22 | var data = iconv.decode(res.data, 'gbk'); 23 | data.should.containEql('天猫'); 24 | res.status.should.equal(200); 25 | }); 26 | 27 | it('should request text ok', function () { 28 | this.timeout(30000); 29 | var res = urllib.request('http://npm.taobao.org', { 30 | dataType: 'text', 31 | timeout: 30000 32 | }); 33 | res.data.should.containEql('淘宝 NPM 镜像'); 34 | res.status.should.equal(200); 35 | }); 36 | 37 | it('should request json ok', function () { 38 | this.timeout(30000); 39 | var res = urllib.request('http://registry.npm.taobao.org/koa', { 40 | dataType: 'json', 41 | timeout: 30000 42 | }); 43 | res.data.name.should.equal('koa'); 44 | res.status.should.equal(200); 45 | }); 46 | 47 | it('should timeout', function () { 48 | try { 49 | urllib.request('http://npm.taobao.org', { 50 | dataType: 'text', 51 | timeout: 10 52 | }); 53 | throw new Error('should not exec'); 54 | } catch (err) { 55 | err.message.should.match(/timeout/i); 56 | } 57 | }); 58 | 59 | it('should writeFile ok', function () { 60 | this.timeout(4000); 61 | var res = urllib.request('http://www.taobao.com/go/rgn/tmall/header/2014/sub-nav.php', { 62 | writeFile: './tmp' 63 | }); 64 | res.status.should.equal(200); 65 | var data = iconv.decode(fs.readFileSync('./tmp'), 'gbk'); 66 | data.should.containEql('天猫'); 67 | fs.unlinkSync('./tmp'); 68 | }); 69 | 70 | it('should not write file when status 302', function () { 71 | this.timeout(4000); 72 | var res = urllib.request('http://www.taobao.com/not/exist/file/path', { 73 | writeFile: './404file' 74 | }); 75 | res.status.should.equal(302); 76 | res.data.toString().should.match(/302/); 77 | fs.existsSync('./404file').should.equal(false); 78 | }); 79 | }); 80 | }); 81 | --------------------------------------------------------------------------------