├── .editorconfig ├── .gitignore ├── .jshintrc ├── .npmignore ├── .npmrc ├── .travis.yml ├── CONTRIBUTING.md ├── build.js ├── can-ndjson-stream-test.js ├── can-ndjson-stream.js ├── demo ├── can-ndjson-stream.html ├── package.json ├── server.js └── todos.ndjson ├── docs └── can-ndjson-stream.md ├── index.html ├── license.md ├── ndjsonStream.gif ├── package.json ├── readme.md ├── test-sauce-labs.js ├── test.html └── test.js /.editorconfig: -------------------------------------------------------------------------------- 1 | ; Unix-style newlines 2 | [*] 3 | end_of_line = LF 4 | indent_style = tab 5 | trim_trailing_whitespace = false 6 | insert_final_newline = true -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # Compiled binary addons (http://nodejs.org/api/addons.html) 20 | build/Release 21 | 22 | # Dependency directory 23 | # Commenting this out is preferred by some people, see 24 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 25 | node_modules 26 | 27 | # Users Environment Variables 28 | .lock-wscript 29 | 30 | dist/ 31 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "globals": { 3 | "steal": true, 4 | "can": true, 5 | "Zepto": true, 6 | "QUnit": true, 7 | "System": true, 8 | "SimpleDOM": true, 9 | "test": true, 10 | "asyncTest": true, 11 | "expect": true, 12 | "module": true, 13 | "ok": true, 14 | "equal": true, 15 | "notEqual": true, 16 | "deepEqual": true, 17 | "notDeepEqual": true, 18 | "strictEqual": true, 19 | "notStrictEqual": true, 20 | "raises": true, 21 | "start": true, 22 | "stop": true, 23 | "global": true, 24 | "ReadableStream": true 25 | }, 26 | "curly": true, 27 | "eqeqeq": true, 28 | "freeze": true, 29 | "indent": 2, 30 | "latedef": true, 31 | "noarg": true, 32 | "undef": true, 33 | "unused": true, 34 | "trailing": true, 35 | "maxdepth": 4, 36 | "boss" : true, 37 | "eqnull": true, 38 | "evil": true, 39 | "loopfunc": true, 40 | "smarttabs": true, 41 | "maxerr" : 200, 42 | "jquery": true, 43 | "dojo": true, 44 | "mootools": true, 45 | "yui": true, 46 | "browser": true, 47 | "phantom": true, 48 | "rhino": true, 49 | "node": true, 50 | "esnext": true 51 | } 52 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | !dist/ -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | package-lock=false 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: xenial 3 | language: node_js 4 | node_js: 5 | - '10' 6 | addons: 7 | chrome: stable 8 | sauce_connect: true 9 | apt: 10 | packages: 11 | - xvfb 12 | before_script: 13 | - 'export DISPLAY='':99.0''' 14 | - 'Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &' 15 | - npm run http-server & 16 | - sleep 3 17 | - sudo chown root /opt/google/chrome/chrome-sandbox 18 | - sudo chmod 4755 /opt/google/chrome/chrome-sandbox 19 | script: npm run ci 20 | env: 21 | global: 22 | - secure: >- 23 | a7ksz7FbsFIH1iLlx7SFGNv4A3ovFrDNowtfFIFl1Ns193vjlo+nNw7LoR3pscUpSqUzz9bgwsL36TeS4wX6mtKivHybke7j02XuIfr14tK2GckD3C2zVbxK2Qvhcfx39nHxweJ7t43xyDS+nbFL+bJnlpaNT8yYIlWWdgaSdEGttuiXsGYN9Lf/qgiwBR9fyfi90hNFmWHzp1t36H9+f9tjIEvYuv779d1Xaodg4kxv984ZIQJHzQ0n0KFKnsG4djnQ2aRBATuIq9A874VxFG9fr6MbnSgIu9mdVQmaqbxWICeI3iJwhjYvx/iloQyJ+vu2ZM6FeAexZXdr1WbnfyrsroKkE7jlbJIuzhDQPj+Mo3cW/DfOBEf4xN4v/4o7XNE8/Y1nubEcKkOnJmVRsprLY5NYUsjsu5yPx/UQ1eOkqo3fYIj/hotCar/53ZFhXlNunGnod3QWMagydpFmrY8cprPPrL8YinjWsH8wo9Q+hFFJLkcaIdF4I4fyGIdbRlrNoLEV7/YJG25KwNIurRwKe+D/EoHWUeJpwf8kI89nZH/RA1ioqZSoZqRWBP4Ayddym4wtaattgfjrO12DcuGUDn0hSBcC+Xyz6DecIeVGMDRn/EUUP7mzuBhXs8Fyh88WdyWMZHWW1fXKe43C7vOUh2Z8ehSjcKLw4rQwoTE= 24 | - secure: >- 25 | RtLiWAWw7XOnysLUCGP7GLySvp7ZbTp/nlyuVJACXG1p+P2PMbSjB4D0uTAAoirnicYP9sQtsZo4OOVnIEdAiK203n4oDvVtt3UEmCt+0NcrrRnapxDJn7dbBeWqlV0yV3uVZq+6gsKYSLJzlGH4j12z/XSMRwtZfj0f+5jG2HzjAQpSbHhbNkl8wdkYqxWwhnDy4VfceII56XHzx1570F0QJmlOTrMBRFJ8FUbNQ8sxHE3DdfNzpUqtl1JPwPLL0oiYlUE8YCKA8NIsKa0YHmpBfyAU903k+Xwa41icxiRiX8/5QYaDEaFtS2a0b55djiEKiKJQy/QahtVVw9MFgTJ5vrPKULi6eifgD5kPbQb2hZ3UsMPCi8QOnXFOtFwj7QnR64lkOqliyiuvdq2NQHOzDN3yY2GU7jB4rYEynjOi1zn+rz2A53tUIEQGw3B1TUlMjvU8fRoZmLTXvizjKpi44Hz8nIA4wdpAD9KvzMBFQ0DE8nfziQ549Q5mlEO2zhrB5oaAkGygqZdToZ0P0FJrhqRcLKaY0jnJUNYU9rFmifwP8SN/jcfHjSiV6Afz1TRWLYPGytWehENp0+7MimCup9XxjASzHmfbLS440zwHtL7thdmDoee9ABvM7FDeeb5wmq+iceuy2DKkpdwYG+Jn0PfuenNKvFtO6hAwj88= 26 | services: 27 | - xvfb 28 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to can-ndjson-stream 2 | 3 | Check out the [contribution guide on CanJS.com](https://canjs.com/doc/guides/contribute.html) for information on: 4 | 5 | - [Code of Conduct](https://canjs.com/doc/guides/contribute.html#CodeofConduct) 6 | - [Getting Help](https://canjs.com/doc/guides/contribute.html#GettingHelp) 7 | - [Project Organization](https://canjs.com/doc/guides/contributing/project-organization.html) 8 | - [Reporting Bugs](https://canjs.com/doc/guides/contributing/bug-report.html) 9 | - [Suggesting Features](https://canjs.com/doc/guides/contributing/feature-suggestion.html) 10 | - [Finding Ways to Contribute](https://canjs.com/doc/guides/contributing/finding-ways-to-contribute.html) 11 | 12 | The rest of this guide has information that’s specific to this repository. 13 | 14 | ## Developing Locally 15 | 16 | This section will walk you through setting up the [repository](https://github.com/canjs/can-ndjson-stream) on your computer. 17 | 18 | ### Signing up for GitHub 19 | 20 | If you don’t already have a GitHub account, you’ll need to [create a new one](https://help.github.com/articles/signing-up-for-a-new-github-account/). 21 | 22 | ### Forking & cloning the repository 23 | 24 | A “fork” is a copy of a repository in your personal GitHub account. “Cloning” is the process of getting the repository’s source code on your computer. 25 | 26 | GitHub has a guide for [forking a repo](https://help.github.com/articles/fork-a-repo/). To fork can-ndjson-stream, you can start by going to its [fork page](https://github.com/canjs/can-ndjson-stream/fork). 27 | 28 | Next, you’ll want to clone the repo. [GitHub’s cloning guide](https://help.github.com/articles/cloning-a-repository/) explains how to do this on Linux, Mac, or Windows. 29 | 30 | GitHub’s guide will [instruct you](https://help.github.com/articles/fork-a-repo/#step-2-create-a-local-clone-of-your-fork) to clone it with a command like: 31 | 32 | ```shell 33 | git clone https://github.com/YOUR-USERNAME/can-ndjson-stream 34 | ``` 35 | 36 | Make sure you replace `YOUR-USERNAME` with your GitHub username. 37 | 38 | ### Installing the dependencies 39 | 40 | After you’ve forked & cloned the repository, you’ll need to install the project’s dependencies. 41 | 42 | First, make sure you’ve [installed Node.js and npm](https://docs.npmjs.com/getting-started/installing-node). 43 | 44 | If you just cloned the repo from the command line, you’ll want to switch to the folder with your clone: 45 | 46 | ```shell 47 | cd can-ndjson-stream 48 | ``` 49 | 50 | Next, install the project’s dependencies with npm: 51 | 52 | ```shell 53 | npm install 54 | ``` 55 | 56 | ### Starting the development server 57 | 58 | Run the following to start a dev server: 59 | 60 | ```shell 61 | npm run develop 62 | ``` 63 | 64 | ### Running the tests 65 | 66 | You can manually run this repository’s tests in any browser by starting the dev server (see the section above) and visiting this page: http://localhost:8080/test/test.html 67 | 68 | Firefox is used to run the repository’s automated tests from the command line. If you don’t already have it, [download Firefox](https://www.mozilla.org/en-US/firefox/new/). Mozilla has guides for installing it on [Linux](https://support.mozilla.org/t5/Install-and-Update/Install-Firefox-on-Linux/ta-p/2516), [Mac](https://support.mozilla.org/t5/Install-and-Update/How-to-download-and-install-Firefox-on-Mac/ta-p/3453), and [Windows](https://support.mozilla.org/t5/Install-and-Update/How-to-download-and-install-Firefox-on-Windows/ta-p/2210). 69 | 70 | After Firefox is installed, you can run: 71 | 72 | ```shell 73 | npm test 74 | ``` 75 | 76 | ### Making a build 77 | 78 | Run the following command to create a build: 79 | 80 | ```shell 81 | npm run build 82 | ``` 83 | 84 | This will create a `dist/` folder that contains the AMD, CommonJS, and global module versions of the project. 85 | 86 | -------------------------------------------------------------------------------- /build.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var stealTools = require("steal-tools"); 3 | 4 | stealTools.export({ 5 | steal: { 6 | config: __dirname + "/package.json!npm" 7 | }, 8 | outputs: { 9 | "+cjs": {}, 10 | "+amd": {}, 11 | "+standalone": { 12 | exports: { 13 | "can-namespace": "can" 14 | } 15 | } 16 | } 17 | }).catch(function(e){ 18 | 19 | setTimeout(function(){ 20 | throw e; 21 | }, 1); 22 | 23 | }); 24 | -------------------------------------------------------------------------------- /can-ndjson-stream-test.js: -------------------------------------------------------------------------------- 1 | var QUnit = require("steal-qunit"); 2 | var ndjsonStream = require("can-ndjson-stream"); 3 | 4 | // Skip all tests in browsers that do not support ReadableStream 5 | // https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream 6 | var isReadStreamSupported = true; 7 | try { 8 | new ReadableStream(); 9 | } catch(err) { 10 | isReadStreamSupported = false; 11 | } 12 | 13 | var conditionalTest = isReadStreamSupported ? QUnit.test : QUnit.skip; 14 | var conditionalAsyncTest = isReadStreamSupported ? QUnit.test : QUnit.skip; 15 | 16 | function readableStreamFromString(s) { 17 | return new ReadableStream({ 18 | start: function(controller) { 19 | var encoder = new TextEncoder(); 20 | // Our current position in s 21 | var pos = 0; 22 | // How much to serve on each push 23 | var chunkSize = 1; 24 | 25 | function push() { 26 | // Are we done? 27 | if (pos >= s.length) { 28 | controller.close(); 29 | return; 30 | } 31 | 32 | // Push some of the html, 33 | // converting it into an Uint8Array of utf-8 data 34 | controller.enqueue( 35 | encoder.encode(s.slice(pos, pos + chunkSize)) 36 | ); 37 | 38 | // Advance the position 39 | pos += chunkSize; 40 | 41 | push(); 42 | } 43 | 44 | // Let's go! 45 | push(); 46 | }, 47 | cancel: function() { 48 | 49 | } 50 | }); 51 | } 52 | 53 | function inputStream(objArray) { 54 | var jsons = objArray.map( function(obj) {return JSON.stringify(obj);} ); 55 | return readableStreamFromString(jsons.join('\n')); 56 | } 57 | 58 | QUnit.module('can-ndjson-stream'); 59 | 60 | conditionalTest('Initialized the plugin', function(assert){ 61 | assert.equal(typeof ndjsonStream, 'function'); 62 | }); 63 | 64 | conditionalAsyncTest('simple_test_from_stream', function(assert) { 65 | 66 | var testObject = [ 67 | {"date":"2017-02-24 03:07:45","user":"21109850","fuel":"37","ammo":"2","steel":"13","baux":"5","seaweed":"0","type":"LOOT","product":"134"}, 68 | {"date":"2017-02-22 04:40:13","user":"21109850","fuel":"37","ammo":"2","steel":"13","baux":"5","seaweed":"0","type":"LOOT","product":"75"}, 69 | {"date":"2017-02-21 20:47:51","user":"26464462","fuel":"37","ammo":"3","steel":"19","baux":"5","seaweed":"1","type":"LOOT","product":"81"} 70 | ]; 71 | var readObjects = []; 72 | var todoStream = ndjsonStream( inputStream(testObject) ); 73 | var reader = todoStream.getReader(); 74 | 75 | var done = assert.async(); 76 | 77 | reader.read().then(function read(result) { 78 | if (result.done) { 79 | assert.deepEqual(readObjects, testObject, "Two arrays should be the same in value"); 80 | done(); 81 | return; 82 | } 83 | readObjects.push(result.value); 84 | 85 | return reader.read().then(read); 86 | }); 87 | }); 88 | 89 | conditionalAsyncTest('malformed json', function(assert) { 90 | var malformed_string = "{\"1\":2}\n{sss: 2}"; 91 | var readObjects = []; 92 | var todoStream = ndjsonStream( readableStreamFromString(malformed_string) ); 93 | var reader = todoStream.getReader(); 94 | var errorCaught = false; 95 | function errCheck() { 96 | errorCaught = true; 97 | } 98 | 99 | // Firefox has a bug where an unhandled rejection is emitted from `new ReadableStream(source)` if `source.start()` rejects. 100 | // See https://bugzilla.mozilla.org/show_bug.cgi?id=1561911 for more info. 101 | // This line should be removed if the Firefox bug above is fixed. 102 | QUnit.onUnhandledRejection = function() {}; 103 | 104 | var done = assert.async(); 105 | var allDone = reader.read().then(function read(result) { 106 | if (result.start) { 107 | return; 108 | } 109 | readObjects.push(result.value); 110 | return reader.read().then(read, errCheck); 111 | }, errCheck); 112 | 113 | allDone.then(function(){ 114 | assert.strictEqual(errorCaught, true, "malformed json string should cause an error"); 115 | done(); 116 | }, function(){ 117 | 118 | assert.strictEqual(errorCaught, true, "rejected: malformed json string should cause an error"); 119 | done(); 120 | }); 121 | 122 | }); 123 | -------------------------------------------------------------------------------- /can-ndjson-stream.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | /*exported ndjsonStream*/ 3 | 4 | var namespace = require('can-namespace'); 5 | 6 | var ndjsonStream = function(response) { 7 | // For cancellation 8 | var is_reader, cancellationRequest = false; 9 | return new ReadableStream({ 10 | start: function(controller) { 11 | var reader = response.getReader(); 12 | is_reader = reader; 13 | var decoder = new TextDecoder(); 14 | var data_buf = ""; 15 | 16 | reader.read().then(function processResult(result) { 17 | if (result.done) { 18 | if (cancellationRequest) { 19 | // Immediately exit 20 | return; 21 | } 22 | 23 | data_buf = data_buf.trim(); 24 | if (data_buf.length !== 0) { 25 | try { 26 | var data_l = JSON.parse(data_buf); 27 | controller.enqueue(data_l); 28 | } catch(e) { 29 | controller.error(e); 30 | return; 31 | } 32 | } 33 | controller.close(); 34 | return; 35 | } 36 | 37 | var data = decoder.decode(result.value, {stream: true}); 38 | data_buf += data; 39 | var lines = data_buf.split("\n"); 40 | for(var i = 0; i < lines.length - 1; ++i) { 41 | var l = lines[i].trim(); 42 | if (l.length > 0) { 43 | try { 44 | var data_line = JSON.parse(l); 45 | controller.enqueue(data_line); 46 | } catch(e) { 47 | controller.error(e); 48 | cancellationRequest = true; 49 | reader.cancel(); 50 | return; 51 | } 52 | } 53 | } 54 | data_buf = lines[lines.length-1]; 55 | 56 | return reader.read().then(processResult); 57 | }); 58 | 59 | }, 60 | cancel: function(reason) { 61 | console.log("Cancel registered due to ", reason); 62 | cancellationRequest = true; 63 | is_reader.cancel(); 64 | } 65 | }); 66 | }; 67 | 68 | module.exports = namespace.ndjsonStream = ndjsonStream; 69 | -------------------------------------------------------------------------------- /demo/can-ndjson-stream.html: -------------------------------------------------------------------------------- 1 | 39 | -------------------------------------------------------------------------------- /demo/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "can-ndjson-stream-demo", 3 | "version": "1.0.0", 4 | "description": "Demo with ndjson server for can-ndjson-stream module which parses a stream of ndjson data into a stream of JS objects.", 5 | "main": "server.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "start": "node server.js" 9 | }, 10 | "keywords": [ 11 | "ndjson", 12 | "stream", 13 | "ReadableStream", 14 | "can-ndjson-stream", 15 | "canjs" 16 | ], 17 | "author": { 18 | "name": "Bitovi", 19 | "email": "contact@bitovi.com", 20 | "url": "http://bitovi.com" 21 | }, 22 | "license": "ISC", 23 | "dependencies": { 24 | "can-ndjson-stream": "0.0.1", 25 | "express": "^4.15.3", 26 | "fs": "0.0.1-security", 27 | "ndjson": "^1.5.0" 28 | }, 29 | "devDependencies": { 30 | "steal": "^1.5.4", 31 | "steal-tools": "^1.3.5" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /demo/server.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var express = require('express'); 3 | var fs = require('fs'); 4 | var ndjson = require('ndjson'); 5 | 6 | var app = express(); 7 | 8 | app.use(express.static(__dirname + '/..')); 9 | 10 | app.get('/api', function(req, res) { 11 | var chunks = []; 12 | var readStream = fs.createReadStream(__dirname + '/todos.ndjson').pipe(ndjson.parse()); 13 | 14 | res.writeHead(200, {'Content-Type': 'application/ndjson'}); 15 | 16 | readStream.on('data', function(data){ 17 | chunks.push(JSON.stringify(data)); 18 | }); 19 | 20 | readStream.on('end', function(){ 21 | var id = setInterval(function(){ 22 | if (chunks.length) { 23 | res.write(chunks.shift() + '\n'); 24 | } else { 25 | clearInterval(id); 26 | res.end(); 27 | } 28 | }, 500); 29 | }); 30 | 31 | }); 32 | 33 | app.listen(8080, function() { 34 | console.log('Checkout demo on http://localhost:8080/demo/can-ndjson-stream.html'); 35 | }); 36 | -------------------------------------------------------------------------------- /docs/can-ndjson-stream.md: -------------------------------------------------------------------------------- 1 | @module {function} can-ndjson-stream 2 | @parent can-data-modeling 3 | @collection can-ecosystem 4 | @package ../package.json 5 | 6 | @description Parses an [NDJSON](http://www.ndjson.org) stream into a stream of JavaScript objects. 7 | 8 | @signature `ndjsonStream(stream)` 9 | 10 | The `can-ndjson-stream` module converts a stream of NDJSON to a [ReadableStream](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream) of JavaScript objects. It is likely that you would use this module to parse an NDJSON stream `response` object received from a [`fetch`](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) request to a service that sends NDJSON streams. 11 | ```js 12 | import ndjsonStream from "can-ndjson-stream"; 13 | 14 | fetch( "/some/endpoint" ) // make a fetch request to a NDJSON stream service 15 | .then( ( response ) => { 16 | return ndjsonStream( response.body ); //ndjsonStream parses the response.body 17 | 18 | } ).then( ( exampleStream ) => { 19 | const reader = exampleStream.getReader(); 20 | let read; 21 | reader.read().then( read = ( result ) => { 22 | if ( result.done ) { 23 | return; 24 | } 25 | 26 | console.log( result.value ); 27 | reader.read().then( read ); 28 | 29 | } ); 30 | } ); 31 | ``` 32 | 33 | @param {ReadableStream} stream A readable [NDJSON](http://www.ndjson.org/) byte stream. 34 | 35 | @return {ReadableStream} The output is a [ReadableStream](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream) that has the following methods: 36 | - getReader() 37 | - cancel([optional cancellation message]) 38 | 39 | @body 40 | 41 | ## Use 42 | 43 | This module is typically used with [fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) to parse an NDJSON response stream. Follow the steps below to use `fetch` with an NDJSON stream service. See the [Creating an NDJSON stream service with NodeJS](#CreatinganNDJSONstreamservicewithNodeJS_) section below to learn how to create a service that emits an NDJSON stream. 44 | 45 | 46 | Assuming your raw data looks something like this: 47 | 48 | ``` 49 | {"item":"first"}\n 50 | {"item":"second"}\n 51 | {"item":"third"}\n 52 | {"item":"fourth"}\n 53 | ``` 54 | 55 | follow these steps to make a request from an NDJSON service at `/some/endpoint`: 56 | 57 | 1. Make a `fetch` request to an NDJSON service by passing the endpoint as an argument. 58 | 2. The service responds with a stream with each value being one line of NDJSON: `{"item":"first"}\n` 59 | 3. `Fetch`'s `then` method is provided a `Response` instance, which we can parse using `ndjsonStream()` into a JavaScript `ReadableStream`. 60 | 5. Each JavaScript object in the stream can be read by calling `[streamName].getReader.read()`, which returns a promise. 61 | 6. The result of that promise will be one JS object from your NDJSON: `{item: "first"}` 62 | 63 | ```js 64 | import ndjsonStream from "can-ndjson-stream"; 65 | 66 | fetch( "/some/endpoint" ) // make a fetch request to a NDJSON stream service 67 | .then( ( response ) => { 68 | return ndjsonStream( response.body ); //ndjsonStream parses the response.body 69 | } ).then( ( exampleStream ) => { 70 | 71 | //retain access to the reader so that you can cancel it 72 | const reader = exampleStream.getReader(); 73 | let read; 74 | 75 | reader.read().then( read = ( result ) => { 76 | if ( result.done ) { 77 | return; 78 | } 79 | console.log( result.value ); //logs {item:"first"} 80 | exampleStream.getReader().read().then( read ); 81 | } ); 82 | } ); 83 | ``` 84 | ## What is NDJSON? 85 | 86 | [NDJSON](http://ndjson.org) is a data format that is separated into individual JSON objects with a newline character (`\n`). The 'nd' stands for newline delimited JSON. Essentially, you have some data that is formatted like this: 87 | 88 | ``` 89 | {"item":"first"}\n 90 | {"item":"second"}\n 91 | {"item":"third"}\n 92 | {"item":"fourth"}\n 93 | ``` 94 | Each item above is separated with a newline and each of those can be sent individually over a stream which allows the client to receive and process the data in specified increments. 95 | 96 | ## Creating an NDJSON stream service with NodeJS. 97 | 98 | This is a quick start guide to getting a NDJSON stream API up and running. 99 | It reads from a local `todos.ndjson` file and responds with a line from the 100 | file every 500ms. 101 | 102 | 1. Install dependencies: 103 | ```bash 104 | $ npm i express path fs ndjson 105 | ``` 106 | 107 | 2. Create a server.js file and copy this code: 108 | 109 | ```js 110 | // server.js 111 | import express from "express"; 112 | 113 | const app = express(); 114 | import path from "path"; 115 | import fs from "fs"; 116 | import ndjson from "ndjson"; 117 | 118 | app.use( express.static( path.join( __dirname, "public" ) ) ); 119 | 120 | app.get( "/", ( req, res ) => { 121 | let readStream = fs.createReadStream( __dirname + "/todos.ndjson" ).pipe( ndjson.parse() ); 122 | 123 | readStream.on( "data", ( data ) => { 124 | chunks.push( JSON.stringify( data ) ); 125 | } ); 126 | 127 | readStream.on( "end", () => { 128 | const id = setInterval( () => { 129 | if ( chunks.length ) { 130 | res.write( chunks.shift() + "\n" ); 131 | } else { 132 | clearInterval( id ); 133 | res.end(); 134 | } 135 | }, 500 ); 136 | } ); 137 | } ); 138 | 139 | app.listen( 3000, () => { 140 | console.log( "Example app listening on port 3000!" ); 141 | } ); 142 | ``` 143 | We use a `setInterval` to slow the stream down so that you can see the stream in action. Feel free to remove the setInterval and use a `while` loop to remove the delay. 144 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | can-ndjson-stream - A Plugin for DoneJS 5 | 17 | 18 | 19 |

Created with the

20 | 21 | DoneJS logo 22 | 23 | plugin generator 24 | 25 | 26 | -------------------------------------------------------------------------------- /license.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright 2017 Justin Meyer (justinbmeyer@gmail.com), Fang Lu (cc2lufang@gmail.com), Siyao Wu (wusiyao@umich.edu), Shang Jiang (mrjiangshang@gmail.com) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /ndjsonStream.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/canjs/can-ndjson-stream/588f44ec9b1ab6f8e6a3d09cac4054516fdcdc5a/ndjsonStream.gif -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "can-ndjson-stream", 3 | "version": "1.0.2", 4 | "description": "", 5 | "homepage": "", 6 | "repository": { 7 | "type": "git", 8 | "url": "git://github.com/canjs/can-ndjson-stream.git" 9 | }, 10 | "author": { 11 | "name": "Bitovi", 12 | "email": "contact@bitovi.com", 13 | "url": "http://bitovi.com" 14 | }, 15 | "scripts": { 16 | "preversion": "npm test && npm run build", 17 | "version": "git commit -am \"Update version number\" && git checkout -b release && git add -f dist/", 18 | "postpublish": "git push --tags && git checkout master && git branch -D release && git push", 19 | "testee-local": "DEBUG=testee:* testee test.html --browsers chrome", 20 | "test": "npm run detect-cycle && npm run jshint && node test.js", 21 | "test-local": "npm run jshint && npm run testee-local", 22 | "jshint": "jshint ./*.js --config", 23 | "release:patch": "npm version patch && npm publish", 24 | "release:minor": "npm version minor && npm publish", 25 | "release:major": "npm version major && npm publish", 26 | "build": "node build.js", 27 | "develop": "done-serve --static --develop --port 8080", 28 | "demo": "cd demo/ && npm i && npm run start", 29 | "http-server": "http-server -p 3000 --silent", 30 | "ci": "npm run build && npm run test && node ./test-sauce-labs.js", 31 | "detect-cycle": "detect-cyclic-packages --ignore done-serve" 32 | }, 33 | "main": "can-ndjson-stream", 34 | "keywords": [], 35 | "steal": { 36 | "main": "can-ndjson-stream", 37 | "configDependencies": [ 38 | "live-reload" 39 | ], 40 | "npmIgnore": [ 41 | "testee", 42 | "generator-donejs", 43 | "donejs-cli", 44 | "steal-tools" 45 | ] 46 | }, 47 | "devDependencies": { 48 | "detect-cyclic-packages": "^1.1.0", 49 | "http-server": "^0.11.0", 50 | "jshint": "^2.9.1", 51 | "saucelabs": "^2.1.9", 52 | "steal": "^1.0.5", 53 | "steal-qunit": "^2.0.0", 54 | "steal-tools": "^2.2.2", 55 | "test-saucelabs": "0.0.6", 56 | "testee": "^0.9.1" 57 | }, 58 | "licenses": [ 59 | { 60 | "type": "MIT", 61 | "url": "http://opensource.org/licenses/MIT" 62 | } 63 | ], 64 | "false": {}, 65 | "dependencies": { 66 | "can-namespace": "^1.0.0" 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # can-ndjson-stream 2 | 3 | [![Join our Slack](https://img.shields.io/badge/slack-join%20chat-611f69.svg)](https://www.bitovi.com/community/slack?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 4 | [![Join our Discourse](https://img.shields.io/discourse/https/forums.bitovi.com/posts.svg)](https://forums.bitovi.com/?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 5 | [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/canjs/can-ndjson-stream/blob/master/license.md) 6 | [![npm version](https://badge.fury.io/js/can-ndjson-stream.svg)](https://www.npmjs.com/package/can-ndjson-stream) 7 | [![Travis build status](https://travis-ci.org/canjs/can-ndjson-stream.svg?branch=master)](https://travis-ci.org/canjs/can-ndjson-stream) 8 | [![Greenkeeper badge](https://badges.greenkeeper.io/canjs/can-ndjson-stream.svg)](https://greenkeeper.io/) 9 | 10 | Most web applications encounter problems of latency because they process data discretely instead of in streams. `ndjsonstream()` converts a ReadableStream of raw ndjson data into a ReadableStream of Javascript objects. 11 | 12 | ## Documentation 13 | 14 | Read the [can-ndjson-stream API docs on CanJS.com](https://canjs.com/doc/can-ndjson-stream.html). 15 | 16 | ## Changelog 17 | 18 | See the [latest releases on GitHub](https://github.com/canjs/can-ndjson-stream/releases). 19 | 20 | ## Contributing 21 | 22 | The [contribution guide](https://github.com/canjs/can-ndjson-stream/blob/master/CONTRIBUTING.md) has information on getting help, reporting bugs, developing locally, and more. 23 | 24 | ## License 25 | 26 | [MIT](https://github.com/canjs/can-ndjson-stream/blob/master/license.md) 27 | -------------------------------------------------------------------------------- /test-sauce-labs.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | var testSauceLabs = require("test-saucelabs"); 4 | 5 | // https://github.com/SeleniumHQ/selenium/wiki/DesiredCapabilities 6 | var platforms = [ 7 | { 8 | browserName: "Safari", 9 | "appium-version": "1.12.1", 10 | platformName: "iOS", 11 | platformVersion: "12.2", 12 | deviceName: "iPhone XS Simulator" 13 | }, 14 | { 15 | browserName: "safari", 16 | platform: "OS X 10.13", 17 | version: "11" 18 | }, 19 | { 20 | browserName: "firefox", 21 | platform: "Windows 10", 22 | version: "68.0" 23 | }, 24 | { 25 | browserName: "googlechrome", 26 | platform: "OS X 10.12", 27 | version: "latest" 28 | }, 29 | { 30 | browserName: "chrome", 31 | platform: "Windows 7", 32 | version: "73.0" 33 | }, 34 | { 35 | browserName: "chrome", 36 | platform: "Windows 10", 37 | version: "59.0" 38 | } 39 | ]; 40 | 41 | var url = "http://localhost:3000/test.html"; 42 | 43 | testSauceLabs({ 44 | urls: [{ name: "can-ndjson-stream", url: url }], 45 | platforms: platforms 46 | }); 47 | -------------------------------------------------------------------------------- /test.html: -------------------------------------------------------------------------------- 1 | 2 | can-ndjson-stream 3 | 4 |
5 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var testee = require("testee"); 2 | 3 | testee.test(['test.html'], [{ 4 | "os": "windows", 5 | "os_version": "10", 6 | "browser": "chrome", 7 | "browser_version": "latest" 8 | },]).then(function() { 9 | process.exitCode = 0; 10 | }, function() { 11 | process.exitCode = 1; 12 | }); 13 | --------------------------------------------------------------------------------