├── lib ├── adapters │ ├── README.md │ ├── http.js │ └── xhr.js ├── utils.js └── index.js ├── index.js ├── .eslintrc ├── webpack.config.js ├── .gitignore ├── test ├── browser │ ├── index.html │ └── xhr.js ├── write.js ├── helpers │ └── server.js └── http.js ├── README.md ├── LICENSE └── package.json /lib/adapters/README.md: -------------------------------------------------------------------------------- 1 | request options: 2 | 3 | options.data (form data, query string is in the url) 4 | options.method (lower case) 5 | options.url 6 | options.headers -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assign = require('lodash.assign'); 4 | const utils = require('./lib/utils'); 5 | const pipelining = require('./lib/'); 6 | 7 | assign(pipelining, utils); 8 | 9 | module.exports = pipelining; 10 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "node": true, 4 | "es6": true 5 | }, 6 | "globals": { 7 | "_": true, 8 | "Promise": true 9 | }, 10 | "parserOptions": { 11 | "sourceType": "module" 12 | }, 13 | "extends": "airbnb/legacy", 14 | "rules": { 15 | "no-new": 0, 16 | "no-param-reassign": 0, 17 | "func-names": 0, 18 | "no-underscore-dangle": 0, 19 | "max-len": 0 20 | } 21 | } -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const path = require('path'); 4 | const buildPath = path.resolve(__dirname, './test/browser/'); 5 | 6 | const webpackConfig = { 7 | entry: './test/browser/xhr.js', 8 | output: { 9 | path: buildPath, 10 | filename: 'bundle.js' 11 | }, 12 | 13 | module: { 14 | loaders: [{ 15 | test: /./, 16 | loader: 'babel-loader', 17 | query: { 18 | presets: ['es2015'] 19 | } 20 | }] 21 | } 22 | }; 23 | 24 | module.exports = webpackConfig; 25 | -------------------------------------------------------------------------------- /lib/adapters/http.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const request = require('request'); 4 | 5 | function sendRequest(options) { 6 | return request[options.method.toLowerCase()](options.url) 7 | .on('response', response => { 8 | response.on('data', data => { 9 | options.onData(data.toString()); 10 | }); 11 | response.on('end', () => { 12 | process.nextTick(() => { 13 | options.onEnd({ status: response.statusCode }); 14 | }); 15 | }); 16 | }); 17 | } 18 | 19 | module.exports = sendRequest; 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # nyc test coverage 18 | .nyc_output 19 | 20 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 21 | .grunt 22 | 23 | # node-waf configuration 24 | .lock-wscript 25 | 26 | # Compiled binary addons (http://nodejs.org/api/addons.html) 27 | build/Release 28 | 29 | # Dependency directories 30 | node_modules 31 | jspm_packages 32 | 33 | # Optional npm cache directory 34 | .npm 35 | 36 | # Optional REPL history 37 | .node_repl_history 38 | 39 | test/browser/bundle.js 40 | -------------------------------------------------------------------------------- /test/browser/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Tests 4 | 5 | 6 | 7 |
8 | 9 | 10 | 15 | 16 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pipelining 2 | Xhr chunked stream client for the browser and node.js 3 | 4 | ## Install 5 | 6 | ```shell 7 | $ npm install pipelining 8 | ``` 9 | 10 | ## Usage 11 | 12 | Client (browser or node.js) 13 | 14 | ```javascript 15 | const pipelining = require('pipelining'); 16 | 17 | const reader = pipelining('/test'); 18 | 19 | function handle(data) { 20 | console.log(data); 21 | } 22 | 23 | function read() { 24 | reader.read().then(partial => { 25 | if (partial.done) { 26 | return; 27 | } 28 | 29 | handle(partial.data).then(read); 30 | }); 31 | } 32 | 33 | read() 34 | ``` 35 | 36 | Server 37 | 38 | ```javascript 39 | const pipelining = require('pipelining'); 40 | // http handler 41 | function (req, res) { 42 | res.write(pipelining.pack(1)); 43 | // after several seconds.. 44 | res.write(pipelining.pack({ tom: 'test' })); 45 | // after 1 min.. 46 | res.end(); 47 | } 48 | ``` 49 | -------------------------------------------------------------------------------- /lib/adapters/xhr.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | function request(options) { 4 | const xhr = new XMLHttpRequest(); 5 | 6 | let status; 7 | let receivedLength = 0; 8 | 9 | xhr.withCredentials = options.withCredentials; 10 | xhr.open(options.method, options.url); 11 | xhr.onreadystatechange = function () { 12 | if (xhr.readyState === 3) { 13 | const text = xhr.responseText; 14 | const partial = text.slice(receivedLength); 15 | receivedLength = text.length; 16 | options.onData(partial); 17 | } 18 | 19 | if (xhr.readyState === 4) { 20 | status = xhr.status || 0; 21 | 22 | const text = xhr.responseText; 23 | const partial = text.slice(receivedLength); 24 | if (partial.length > 0) { 25 | options.onData(partial); 26 | } 27 | 28 | options.onEnd({ status }); 29 | } 30 | }; 31 | 32 | xhr.send(options.data); 33 | return xhr; 34 | } 35 | 36 | module.exports = request; 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 TomWan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/utils.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const regexp = /^(\d+):(.*)$/; 4 | 5 | function pack(target) { 6 | let str = JSON.stringify({ data: target }); 7 | return str.length + ':' + str + '\r\n'; 8 | } 9 | 10 | // Return list 11 | function unpack(str) { 12 | const str2 = str.slice(0, -2); 13 | const splitted = str2.split('\r\n'); 14 | 15 | return splitted.map(item => { 16 | const matches = item.match(regexp); 17 | if (!matches) { 18 | throw new Error('pipelining format error:' + item); 19 | } 20 | const s = matches[2]; 21 | return JSON.parse(s).data; 22 | }); 23 | } 24 | 25 | function isValid(str) { 26 | if (!str) { 27 | return false; 28 | } 29 | 30 | if (str.slice(-2) !== '\r\n') { 31 | return false; 32 | } 33 | 34 | const str2 = str.slice(0, -2); 35 | const splitted = str2.split('\r\n'); 36 | for (let i = 0; i < splitted.length; i += 1) { 37 | const matches = splitted[i].match(regexp); 38 | if (!matches) { 39 | return false; 40 | } 41 | if (Number(matches[1]) !== matches[2].length) { 42 | return false; 43 | } 44 | } 45 | 46 | return true; 47 | } 48 | 49 | module.exports.pack = pack; 50 | module.exports.unpack = unpack; 51 | module.exports.isValid = isValid; 52 | -------------------------------------------------------------------------------- /test/browser/xhr.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const pipelining = require('../../'); 4 | 5 | function read(time) { 6 | let tmp = 0; 7 | const reader = pipelining('/', { timeout: 2000 }); 8 | let promise = reader.read(); 9 | 10 | while (tmp < time) { 11 | tmp += 1; 12 | if (tmp === time) { 13 | return promise; 14 | } 15 | 16 | promise = promise.then(() => { 17 | return reader.read(); 18 | }); 19 | } 20 | } 21 | 22 | describe('pipelining', function () { 23 | this.timeout(30000); 24 | 25 | it('Get 1st partial correctly', function (done) { 26 | read(1).then(result => { 27 | expect(result).to.eql({ data: 1 }); 28 | done(); 29 | }); 30 | }); 31 | 32 | it('Get 2nd partial correctly', function (done) { 33 | read(2).then(result => { 34 | expect(result).to.eql({ data: 2 }); 35 | done(); 36 | }); 37 | }); 38 | 39 | it('Get 5th partial correctly', function (done) { 40 | read(5).then(result => { 41 | expect(result).to.eql({ data: 5 }); 42 | done(); 43 | }); 44 | }); 45 | 46 | it('Get error when format error', function (done) { 47 | read(7).then(console.log).catch(e => { 48 | console.log(e.stack); 49 | expect(e.message).to.eql('timeout'); 50 | done(); 51 | }); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pipelining", 3 | "version": "2.2.3", 4 | "description": "xhr chunked stream handler for the browser and NodeJS", 5 | "main": "index.js", 6 | "directories": { 7 | "test": "test" 8 | }, 9 | "scripts": { 10 | "start": "node test/helpers/server", 11 | "test": "mocha && npm run browser-test", 12 | "browser-test": "mocha-phantomjs -p /usr/local/bin/phantomjs http://localhost:9999/test", 13 | "build": "webpack ./webpack.config.js" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "git+https://github.com/wanming/pipelining.git" 18 | }, 19 | "author": "", 20 | "license": "ISC", 21 | "bugs": { 22 | "url": "https://github.com/wanming/pipelining/issues" 23 | }, 24 | "homepage": "https://github.com/wanming/pipelining#readme", 25 | "devDependencies": { 26 | "babel-core": "^6.22.1", 27 | "babel-loader": "^6.2.10", 28 | "babel-preset-es2015": "^6.22.0", 29 | "chai": "^3.5.0", 30 | "mocha": "^3.2.0", 31 | "webpack": "^1.14.0" 32 | }, 33 | "browser": { 34 | "./lib/adapters/http.js": "./lib/adapters/xhr.js" 35 | }, 36 | "dependencies": { 37 | "bluebird": "^3.4.7", 38 | "debug": "^2.6.1", 39 | "lodash.assign": "^4.2.0", 40 | "lodash.clone": "^4.5.0", 41 | "lodash.keys": "^4.2.0", 42 | "request": "^2.79.0" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /test/write.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const expect = require('chai').expect; 4 | 5 | const utils = require('../lib/utils'); 6 | 7 | describe('utils', function () { 8 | describe('pack', function () { 9 | it('pack number correctly', function () { 10 | const a1 = utils.pack(1); 11 | expect(a1).to.equal('10:{"data":1}\r\n'); 12 | }); 13 | 14 | it('pack string correctly', function () { 15 | const a1 = utils.pack({ t: 1 }); 16 | expect(a1).to.equal('16:{"data":{"t":1}}\r\n'); 17 | }); 18 | }); 19 | 20 | describe('unpack', function () { 21 | it('unpack single correctly', function () { 22 | const a1 = utils.unpack('16:{"data":{"t":1}}\r\n'); 23 | expect(a1).to.deep.equal([{ t: 1 }]); 24 | }); 25 | 26 | it('unpack multi correctly', function () { 27 | const a1 = utils.unpack('16:{"data":{"t":1}}\r\n10:{"data":1}\r\n'); 28 | expect(a1).to.deep.equal([{ t: 1 }, 1]); 29 | }); 30 | }); 31 | 32 | describe('isValid', function () { 33 | it('should return invalid when length not matched', function () { 34 | const a1 = utils.isValid('15:{"data":{"t":1}}\r\n'); 35 | expect(a1).to.equal(false); 36 | }); 37 | 38 | it('should return invalid when tail is wrong', function () { 39 | const a1 = utils.isValid('15:{"data":{"t":1}}\r'); 40 | expect(a1).to.equal(false); 41 | }); 42 | 43 | it('should return invalid when regexp not matched', function () { 44 | const a1 = utils.isValid('123123\r\n'); 45 | expect(a1).to.equal(false); 46 | }); 47 | 48 | it('should return invalid when empty', function () { 49 | const a1 = utils.isValid(); 50 | expect(a1).to.equal(false); 51 | }); 52 | 53 | it('should return valid', function () { 54 | const a1 = utils.isValid('16:{"data":{"t":1}}\r\n10:{"data":1}\r\n'); 55 | expect(a1).to.equal(true); 56 | }); 57 | }); 58 | }); -------------------------------------------------------------------------------- /test/helpers/server.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Promise = require('bluebird'); 4 | const path = require('path'); 5 | const http = require('http'); 6 | const fs = require('fs'); 7 | const pipelining = require('../../'); 8 | 9 | const port = 9999; 10 | 11 | // Create an HTTP server 12 | var srv = http.createServer((req, res) => { 13 | if (req.url === '/') { 14 | res.writeHead(200, { 15 | 'Transfer-Encoding': 'chunked', 'Content-Type': 'text/html' 16 | }); 17 | 18 | let n = 0; 19 | 20 | const write = () => { 21 | let delay = n % 2 === 0 ? 0 : parseInt(Math.random() * 1000, 10); 22 | n += 1; 23 | 24 | if (n === 10) { 25 | res.end('33333', 500); 26 | return Promise.resolve(true); 27 | } 28 | 29 | if (n === 6) { 30 | res.write('aaaa'); 31 | } 32 | 33 | res.write(pipelining.pack(n)); 34 | return Promise.delay(delay).then(write); 35 | }; 36 | 37 | write(); 38 | return; 39 | } 40 | 41 | if (req.url === '/test') { 42 | const indexHtml = fs.readFileSync(path.join(process.cwd(), 'test/browser/index.html')).toString(); 43 | res.end(indexHtml); 44 | return; 45 | } 46 | 47 | if (req.url === '/test2') { 48 | res.writeHead(200, { 49 | 'Transfer-Encoding': 'chunked', 'Content-Type': 'text/html' 50 | }); 51 | 52 | setTimeout(() => { 53 | res.write(pipelining.pack('tom1')); 54 | }, 100); 55 | 56 | setTimeout(() => { 57 | res.write(pipelining.pack('tom2')); 58 | }, 120); 59 | 60 | setTimeout(() => { 61 | res.end(); 62 | }, 130); 63 | return; 64 | } 65 | 66 | const filePath = path.join(process.cwd(), req.url); 67 | 68 | if (fs.existsSync(filePath)) { 69 | const content = fs.readFileSync(filePath).toString(); 70 | res.end(content); 71 | return; 72 | } 73 | 74 | res.end('', 404); 75 | }); 76 | 77 | console.log('server listening port', port); 78 | srv.listen(port); 79 | -------------------------------------------------------------------------------- /test/http.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const expect = require('chai').expect; 4 | const pipelining = require('../'); 5 | 6 | function read(time, options) { 7 | let tmp = 0; 8 | const reader = pipelining('http://localhost:9999/', options); 9 | let promise = reader.read(); 10 | 11 | while (tmp < time) { 12 | tmp += 1; 13 | if (tmp === time) { 14 | return promise; 15 | } 16 | 17 | promise = promise.then(() => { 18 | return reader.read(); 19 | }); 20 | } 21 | } 22 | 23 | describe('pipelining', function () { 24 | this.timeout(20000); 25 | 26 | it('Get 1st partial correctly', function (done) { 27 | read(1).then(result => { 28 | expect(result).to.eql({ data: 1 }); 29 | done(); 30 | }); 31 | }); 32 | 33 | it('Get 2nd partial correctly', function (done) { 34 | read(2).then(result => { 35 | expect(result).to.eql({ data: 2 }); 36 | done(); 37 | }); 38 | }); 39 | 40 | it('Get 5th partial correctly', function (done) { 41 | read(5).then(result => { 42 | expect(result).to.eql({ data: 5 }); 43 | done(); 44 | }); 45 | }); 46 | 47 | it('Get error when format error', function (done) { 48 | read(7).catch(e => { 49 | expect(e.message).to.eql('Format error'); 50 | done(); 51 | }); 52 | }); 53 | 54 | it('Done successfully', function (done) { 55 | const reader = pipelining('http://localhost:9999/test2'); 56 | reader.read().then(() => { 57 | return reader.read(); 58 | }).then(() => { 59 | return reader.read(); 60 | }).then((doneResult) => { 61 | expect(doneResult.done).to.eql(true); 62 | done(); 63 | }); 64 | }); 65 | 66 | it('Abort successfully', function (done) { 67 | const reader = pipelining('http://localhost:9999/test2'); 68 | reader.abort(); 69 | reader.read().catch(e => { 70 | expect(e.message).to.eql('Aborted'); 71 | done(); 72 | }); 73 | }); 74 | 75 | it('Throw timeout after 0.5s', function (done) { 76 | read(7, { timeout: 500 }).catch(e => { 77 | expect(e.message).to.eql('Timeout'); 78 | done(); 79 | }); 80 | }); 81 | }); 82 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const keys = require('lodash.keys'); 4 | const urllib = require('url'); 5 | const assign = require('lodash.assign'); 6 | const clone = require('lodash.clone'); 7 | const Promise = require('bluebird'); 8 | const utils = require('./utils'); 9 | const _request = require('./adapters/http'); 10 | const debug = require('debug')('pipelining'); 11 | 12 | function request(_options) { 13 | const options = clone(_options); 14 | options.method = options.method.toUpperCase(); 15 | let data; 16 | 17 | if (options.data) { 18 | const parsedUrl = urllib.parse(options.url, true); 19 | if (options.method === 'GET') { 20 | parsedUrl.query = parsedUrl.query || {}; 21 | assign(parsedUrl.query, options.data); 22 | parsedUrl.search = null; 23 | options.url = urllib.format(parsedUrl); 24 | } else { 25 | data = new FormData(); 26 | keys(options.data).forEach(key => { 27 | data.append(key, options.data[key]); 28 | }); 29 | } 30 | } 31 | 32 | const receivedPartials = []; 33 | 34 | let readCount = 0; 35 | let waitingPromise; 36 | let done = false; 37 | let resolveDone = false; 38 | let error; 39 | let aborted = false; 40 | let doneResult; 41 | let startTime = Date.now(); 42 | 43 | // NGINX will split big data to several parts 44 | let partialString = ''; 45 | 46 | const onData = function (partial) { 47 | debug('get data:', partial); 48 | partial = partialString + partial; 49 | if (!utils.isValid(partial)) { 50 | partialString = partial; 51 | return; 52 | } 53 | 54 | partialString = ''; 55 | 56 | const dataArray = utils.unpack(partial); 57 | 58 | if (dataArray.length > 0) { 59 | dataArray.forEach(item => receivedPartials.push(item)); 60 | 61 | if (waitingPromise) { 62 | waitingPromise.resolve({ data: dataArray[0] }); 63 | waitingPromise = null; 64 | readCount += 1; 65 | } 66 | } 67 | }; 68 | 69 | const onEnd = function (result) { 70 | debug('request ended'); 71 | done = true; 72 | doneResult = { done: true, status: (result || {}).status }; 73 | 74 | if (waitingPromise && receivedPartials.length === readCount) { 75 | waitingPromise.resolve(doneResult); 76 | } 77 | }; 78 | 79 | const requestResult = _request(assign({}, options, { onData, onEnd })); 80 | 81 | const reader = { 82 | abort() { 83 | debug('pipelining aborted'); 84 | aborted = true; 85 | return requestResult.abort(); 86 | }, 87 | read() { 88 | const now = Date.now(); 89 | return new Promise((resolve, reject) => { 90 | if (aborted) { 91 | reject(new Error('Aborted')); 92 | return; 93 | } 94 | 95 | if (error) { 96 | reject(error); 97 | return; 98 | } 99 | 100 | if (done) { 101 | if (resolveDone) { 102 | reject(new Error('Already resolved')); 103 | return; 104 | } 105 | 106 | if (receivedPartials.length === readCount) { 107 | resolve(doneResult); 108 | resolveDone = true; 109 | return; 110 | } 111 | } 112 | 113 | if (receivedPartials.length > readCount) { 114 | resolve({ data: receivedPartials[readCount] }); 115 | readCount += 1; 116 | return; 117 | } 118 | 119 | waitingPromise = { resolve, reject }; 120 | }).timeout(options.timeout - (now - startTime), new Error('Timeout')); 121 | } 122 | }; 123 | 124 | return reader; 125 | } 126 | 127 | function getRequestFunction(method) { 128 | return function (url, _options) { 129 | const options = _options || {}; 130 | return request(assign({}, { url, method, timeout: 60000 }, options)); 131 | }; 132 | } 133 | 134 | module.exports = module.exports.get = getRequestFunction('get'); 135 | module.exports.request = request; 136 | 137 | ['post', 'patch', 'put'].forEach(method => { 138 | module.exports[method] = getRequestFunction(method); 139 | }); 140 | 141 | if (typeof window !== 'undefined') { 142 | window.pipelining = exports; 143 | } 144 | --------------------------------------------------------------------------------