├── .gitignore ├── example.js ├── index.js ├── package.json ├── readme.md └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .npmrc 3 | node_modules 4 | package-lock.json -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | var registry = require('.')() 2 | 3 | registry 4 | .on('package', function (pkg) { 5 | // nice clean package object 6 | }) 7 | .on('up-to-date', function (pkg) { 8 | // consumed all changes (for now) 9 | // The stream will remain open and continue receiving package 10 | // updates from the registry as they occur in real time. 11 | }) 12 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const ChangesStream = require('changes-stream') 2 | const Package = require('nice-package') 3 | const got = require('got') 4 | const EventEmitter = require('events') 5 | const util = require('util') 6 | 7 | function Emitter () { EventEmitter.call(this) } 8 | util.inherits(Emitter, EventEmitter) 9 | 10 | module.exports = function (opts) { 11 | if (!opts) opts = {} 12 | var defaults = { 13 | db: 'https://replicate.npmjs.com', 14 | include_docs: true 15 | } 16 | opts = Object.assign(defaults, opts) 17 | 18 | var emitter = new Emitter() 19 | var finalUpdate 20 | var changes = new ChangesStream(opts) 21 | 22 | got(opts.db, {json: true}).then(response => { 23 | finalUpdate = response.body.update_seq 24 | changes.on('data', function (change) { 25 | var pkg = new Package(change.doc) 26 | if (!pkg.valid) return 27 | emitter.emit('package', pkg, change.seq) 28 | if (change.seq >= finalUpdate) return emitter.emit('up-to-date') 29 | }) 30 | }) 31 | 32 | return emitter 33 | } 34 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "package-stream", 3 | "version": "3.1.0", 4 | "description": "An endless stream of clean package data from the npm registry.", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "node test.js | tap-spec" 8 | }, 9 | "repository": "https://github.com/zeke/package-stream", 10 | "keywords": [ 11 | "npm", 12 | "registry", 13 | "stream", 14 | "packages", 15 | "events", 16 | "normalize", 17 | "package.json" 18 | ], 19 | "author": "Zeke Sikelianos (http://zeke.sikelianos.com)", 20 | "license": "MIT", 21 | "bugs": { 22 | "url": "https://github.com/zeke/package-stream/issues" 23 | }, 24 | "homepage": "https://github.com/zeke/package-stream#readme", 25 | "dependencies": { 26 | "changes-stream": "^1.1.0", 27 | "got": "^6.3.0", 28 | "nice-package": "^3.0.1" 29 | }, 30 | "devDependencies": { 31 | "tap-spec": "^4.1.1", 32 | "tape": "^4.6.0" 33 | }, 34 | "engines": { 35 | "node": ">4" 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # package-stream 2 | 3 | An endless stream of [nice package data](https://github.com/zeke/nice-package) 4 | from the npm registry. 5 | 6 | See also [all-the-packages](https://github.com/zeke/all-the-packages), a similar 7 | package designed for offline use. 8 | 9 | ## Installation 10 | 11 | ```sh 12 | npm install package-stream --save 13 | ``` 14 | 15 | ## Usage 16 | 17 | The stream is an event emitter that emits two events: `package` and `up-to-date`. 18 | The `up-to-date` event is emitted when the stream reaches the end of all 19 | existing packages, but unlike typical read streams, this stream has no `end` 20 | event. It remains open indefinitely, emitting `package` events as new package 21 | versions are published to the npm registry in real time. 22 | 23 | 24 | ```js 25 | const registry = require('package-stream')() 26 | 27 | registry 28 | .on('package', function (pkg, seq) { 29 | // pkg: nice clean package object 30 | // seq: the sequence number of the "package" event, useful for logging/tracking 31 | }) 32 | .on('up-to-date', function () { 33 | // consumed all changes (for now) 34 | // The stream will remain open and continue receiving package 35 | // updates from the registry as they occur in real time. 36 | }) 37 | ``` 38 | 39 | ### Nice Packages 40 | 41 | Each object emitted by the `package` event is a 42 | [nice-package](http://ghub.io/nice-package) instance. 43 | Nice packages have cleaner metadata than you'd get directly from the npm 44 | registry, and some handy 45 | [convenience methods](https://github.com/nice-registry/nice-package#convenience-methods) 46 | . 47 | 48 | Here's an example that uses the 49 | [`somehowDependsOn()`](https://github.com/nice-registry/nice-package#pkgsomehowdependsonpkgname) 50 | method to find all packages the have `choo` in their `dependencies` or 51 | `devDependencies`: 52 | 53 | ```js 54 | const registry = require('package-stream')() 55 | const dependents = [] 56 | 57 | registry 58 | .on('package', function (pkg) { 59 | if (pkg.someHowDependsOn('choo')) dependents.push(pkg.name) 60 | }) 61 | .on('up-to-date', function () { 62 | process.stdout.write(JSON.stringify(dependents)) 63 | process.exit() 64 | }) 65 | ``` 66 | 67 | To see the full list of available methods, check out the 68 | [nice-package documentation](https://github.com/zeke/nice-package/blob/master/README.md#convenience-methods). 69 | 70 | ### Options 71 | 72 | The [`changes-stream`](http://ghub.io/changes-stream) package is used 73 | under the hood, and 74 | [all of its options](https://github.com/jcrugzz/changes-stream#options) 75 | are supported by `package-stream`. The default options used by `package-stream` 76 | are: 77 | 78 | ```js 79 | { 80 | db: 'https://replicate.npmjs.com', 81 | include_docs: true 82 | } 83 | ``` 84 | 85 | The options you provide are merged with the defaults above. 86 | 87 | ## Tests 88 | 89 | ```sh 90 | npm install 91 | npm test 92 | ``` 93 | 94 | ## Dependencies 95 | 96 | - [changes-stream](https://github.com/jcrugzz/changes-stream): Simple module that handles getting changes from couchdb 97 | - [got](https://github.com/sindresorhus/got): Simplified HTTP requests 98 | - [nice-package](https://github.com/zeke/nice-package): Clean up messy package metadata from the npm registry 99 | 100 | ## Dev Dependencies 101 | 102 | - [tap-spec](https://github.com/scottcorgan/tap-spec): Formatted TAP output like Mocha's spec reporter 103 | - [tape](https://github.com/substack/tape): tap-producing test harness for node and browsers 104 | ## License 105 | 106 | MIT 107 | 108 | _Generated by [package-json-to-readme](https://github.com/zeke/package-json-to-readme)_ 109 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | const test = require('tape') 2 | var stream = require('.')() 3 | var somePkg = null 4 | var i = 0 5 | 6 | test('packageStream', function (t) { 7 | stream 8 | .on('package', function (pkg, seq) { 9 | t.equal(typeof seq, 'number', 'has a sequence number') 10 | 11 | t.comment(pkg.name) 12 | if (!pkg.valid) { 13 | console.log(pkg.validationErrors) 14 | } 15 | t.ok(pkg.valid, 'is valid') 16 | 17 | // bail after a while 18 | if (++i > 10 * 1000) done(t) 19 | }) 20 | }) 21 | 22 | function done (t) { 23 | t.end() 24 | process.exit() 25 | } 26 | --------------------------------------------------------------------------------