├── .gitignore ├── lib ├── keyword.js ├── registry.js ├── user.js └── module.js ├── package.json ├── LICENSE.md ├── test.js ├── README.md └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .nyc_output 3 | -------------------------------------------------------------------------------- /lib/keyword.js: -------------------------------------------------------------------------------- 1 | module.exports = function(modules) { 2 | function Keyword(name) { 3 | if (!(this instanceof Keyword)) return new Keyword(name) 4 | this.name = name 5 | } 6 | 7 | Keyword.prototype.count = function(options, callback) { 8 | return modules.get('_design/app/_view/byKeyword', { 9 | group_level: 1 10 | , startkey: [this.name] 11 | , endkey: [this.name, {}] 12 | }, callback) 13 | } 14 | Keyword.prototype.count.select = ['rows', true, 'value'] 15 | Keyword.prototype.count.single = true 16 | 17 | Keyword.prototype.list = function(options, callback) { 18 | return modules.get('_design/app/_view/byKeyword', { 19 | group_level: 2 20 | , startkey: [this.name] 21 | , endkey: [this.name, {}] 22 | }) 23 | } 24 | Keyword.prototype.list.select = ['rows', true] 25 | Keyword.prototype.list.map = function(row) { 26 | return row.key[1] 27 | } 28 | 29 | return Keyword 30 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "npm-stats", 3 | "version": "1.2.0", 4 | "description": "Convenience module for getting back data from an NPM registry", 5 | "main": "index.js", 6 | "dependencies": { 7 | "JSONStream": "~0.6.2", 8 | "event-stream": "~3.0.12", 9 | "lodash.merge": "~3.3.2", 10 | "nano": "^6.2.0", 11 | "request": "~2.45.0", 12 | "stream-reduce": "~1.0.3", 13 | "through": "~2.3.6" 14 | }, 15 | "devDependencies": { 16 | "chai": "^3.5.0", 17 | "moment": "^2.11.1", 18 | "tap": "^5.4.2" 19 | }, 20 | "scripts": { 21 | "test": "tap --coverage ./test.js" 22 | }, 23 | "repository": { 24 | "type": "git", 25 | "url": "git://github.com/hughsk/npm-stats.git" 26 | }, 27 | "keywords": [ 28 | "modules", 29 | "npm", 30 | "data", 31 | "information", 32 | "analytics", 33 | "statistics", 34 | "stats", 35 | "users", 36 | "keywords" 37 | ], 38 | "author": "Hugh Kennedy (http://hughskennedy.com/)", 39 | "license": "MIT" 40 | } 41 | -------------------------------------------------------------------------------- /lib/registry.js: -------------------------------------------------------------------------------- 1 | module.exports = function(modules, downloadUrl, users, mainopts) { 2 | function Registry() { 3 | if (!(this instanceof Registry)) return new Registry(name) 4 | } 5 | 6 | Registry.prototype.list = function(options, callback) { 7 | return modules.get('_design/app/_view/browseAll', { 8 | group_level: 1 9 | }, callback) 10 | } 11 | Registry.prototype.list.select = ['rows', true, 'key', '0'] 12 | 13 | Registry.prototype.listByDate = function(options, callback) { 14 | var params = {} 15 | 16 | if (options.since && typeof options.since !== 'string') 17 | params.startkey = new Date(+options.since) 18 | if (options.until && typeof options.until !== 'string') 19 | params.endkey = new Date(+options.until) 20 | 21 | return modules.get('_design/app/_view/updated', params, callback) 22 | } 23 | Registry.prototype.listByDate.select = ['rows', true] 24 | Registry.prototype.listByDate.map = function(row) { 25 | return { name: row.id, date: row.key } 26 | } 27 | 28 | return Registry 29 | } 30 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | This software is released under the MIT license: 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so, 8 | subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /lib/user.js: -------------------------------------------------------------------------------- 1 | module.exports = function(module, downloadUrl, users, mainopts) { 2 | function User(name) { 3 | if (!(this instanceof User)) return new User(name) 4 | this.name = name 5 | this.key = 'org.couchdb.user:' + this.name 6 | } 7 | 8 | User.prototype.count = function(options, callback) { 9 | return module.get('_design/app/_view/npmTop', { 10 | group_level: 1 11 | , startkey: [this.name] 12 | , endkey: [this.name, {}] 13 | }, callback) 14 | } 15 | User.prototype.count.select = ['rows', true, 'value'] 16 | User.prototype.count.single = true 17 | 18 | User.prototype.list = function(options, callback) { 19 | return module.get('_design/app/_view/byUser', { 20 | startkey: this.name 21 | , endkey: this.name 22 | }) 23 | } 24 | User.prototype.list.select = ['rows', true] 25 | User.prototype.list.map = function(row) { 26 | return row.value 27 | } 28 | 29 | User.prototype.starred = function(options, callback) { 30 | return module.get('_design/app/_view/starredByUser', { 31 | startkey: this.name 32 | , endkey: this.name 33 | }) 34 | } 35 | User.prototype.starred.select = ['rows', true] 36 | User.prototype.starred.map = function(row) { 37 | return row.value 38 | } 39 | 40 | User.prototype.info = function(options, callback) { 41 | return users.get('org.couchdb.user:' + this.name, callback) 42 | } 43 | 44 | return User 45 | } 46 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | /* global describe, it */ 2 | 3 | var npmStats = require('./') 4 | var expect = require('chai').expect 5 | var moment = require('moment') 6 | 7 | require('tap').mochaGlobals() 8 | require('chai').should() 9 | 10 | describe('npm-stats', function () { 11 | describe('registry', function () { 12 | describe('listByDate()', function () { 13 | it('returns modules published before a given date', function (done) { 14 | npmStats().listByDate({since: moment().subtract(1, 'hour').toDate()}, function (err, packages) { 15 | expect(err).to.equal(null) 16 | packages.length.should.be.gt(0) 17 | return done() 18 | }) 19 | }) 20 | }) 21 | }) 22 | 23 | describe('keyword', function () { 24 | describe('count()', function () { 25 | it('returns count of modules using keyword', function (done) { 26 | npmStats().keyword('foobar').count(function (err, count) { 27 | expect(err).to.equal(null) 28 | count.should.be.gt(0) 29 | return done() 30 | }) 31 | }) 32 | }) 33 | }) 34 | 35 | describe('module', function () { 36 | describe('info()', function () { 37 | it('fetches info for module', function (done) { 38 | npmStats().module('lodash').info(function (err, lodash) { 39 | expect(err).to.equal(null) 40 | lodash.name.should.equal('lodash') 41 | return done() 42 | }) 43 | }) 44 | }) 45 | 46 | describe('downloads()', function () { 47 | it('returns download counts for a given module', function (done) { 48 | npmStats().module('lodash').downloads(function (err, downloads) { 49 | expect(err).to.equal(null) 50 | downloads[0].value.should.be.gt(0) 51 | return done() 52 | }) 53 | }) 54 | }) 55 | }) 56 | 57 | describe('user', function () { 58 | describe('count()', function () { 59 | it('fetches count of npm modules', function (done) { 60 | npmStats().user('bcoe').count(function (err, count) { 61 | expect(err).to.equal(null) 62 | count.should.be.gt(0) 63 | return done() 64 | }) 65 | }) 66 | }) 67 | }) 68 | 69 | describe('configuration', function () { 70 | it('allows registry URL to be overridden', function (done) { 71 | npmStats('https://registry.npmjs.org', {modules: ''}).module('lodash').info(function (err, lodash) { 72 | expect(err).to.equal(null) 73 | lodash.name.should.equal('lodash') 74 | return done() 75 | }) 76 | }) 77 | 78 | it('allows registry URL to be overridden with configuration object', function (done) { 79 | npmStats({ 80 | modules: '', 81 | registry: 'https://registry.npmjs.org' 82 | }).module('lodash').info(function (err, lodash) { 83 | expect(err).to.equal(null) 84 | lodash.name.should.equal('lodash') 85 | return done() 86 | }) 87 | }) 88 | }) 89 | }) 90 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # npm-stats # 2 | 3 | Convenience module for getting back data from an NPM registry. 4 | All of the methods return a JSON stream, and/or take a callback. 5 | Where specified, some methods take an optional options object as well. 6 | 7 | ## API ## 8 | 9 | **registry = require('npm-stats')([url, options])** 10 | 11 | Returns a new registry instance, 12 | defaulting to [isaacs.iriscouch.com](https://isaacs.iriscouch.com/). 13 | 14 | Options: 15 | 16 | * `dirty`: pass this as true to disable data cleaning, instead getting 17 | the raw data direct from NPM's CouchDB. 18 | * `modules`: the database to use for retrieving modules. Defaults 19 | to "registry". 20 | * `downloads`: the database to use for retrieving download data. Defaults 21 | to "downloads". 22 | * `users`: the database to use for retrieving users. Defaults to "users". 23 | 24 | **registry.list()** 25 | 26 | Returns an array containing every module currently in the chosen NPM registry. 27 | 28 | **registry.listByDate(options)** 29 | 30 | Get a list of each module in the chosen NPM registry, sorted by date last 31 | updated, in ascending order. 32 | 33 | You can also pass the following options: 34 | 35 | * `since`: only include modules updated since this date. 36 | * `until`: only include modules updated before this date. 37 | 38 | ### Keywords ### 39 | 40 | **registry.keyword(name).count()** 41 | 42 | Get the number of modules using a specific keyword. 43 | 44 | **registry.keyword(name).list()** 45 | 46 | Get a list of modules using a specific keyword. 47 | 48 | ### Users ### 49 | 50 | **registry.user(name).count()** 51 | 52 | Get the number of modules a user has authored. 53 | 54 | **registry.user(name).list()** 55 | 56 | Get a list of the modules a user has authored. 57 | 58 | **registry.user(name).starred()** 59 | 60 | Get a list of the modules a user has starred. 61 | 62 | ### Modules ### 63 | 64 | **registry.module(name).info()** 65 | 66 | Returns the data normally accessible from 67 | `https://registry.npmjs.org/:pkg`. 68 | 69 | **registry.module(name).version(version)** 70 | 71 | Returns the data normally accessible from 72 | `http://registry.npmjs.org/:pkg/:version`. 73 | 74 | **registry.module(name).downloads()** 75 | 76 | Returns a list of download counts for the module, by date, e.g.: 77 | 78 | ``` json 79 | [ 80 | { "date": "2012-12-10", "value": 64 }, 81 | { "date": "2012-12-11", "value": 82 } 82 | ] 83 | ``` 84 | 85 | Days without a download are omitted. Options: 86 | 87 | * `since`: The earliest date to return download info from. 88 | * `until`: The latest date to return download info from. 89 | 90 | **registry.module(name).stars()** 91 | 92 | Returns a list of the users who have starred a module. 93 | 94 | **registry.module(name).dependents()** 95 | 96 | Returns a list of modules that depend on a module. 97 | 98 | **registry.module(name).latest()** 99 | 100 | Returns the latest `package.json` file for a module. 101 | 102 | **registry.module(name).field(name, [callback])** 103 | 104 | Returns a field from the latest `package.json` file for a module. 105 | 106 | **registry.module(name).size()** 107 | 108 | Returns data on the module's size, e.g. 109 | 110 | ``` json 111 | { 112 | "_id": "browserify", 113 | "size": 26338241, 114 | "count": 179, 115 | "avg": 147141.01117318432 116 | } 117 | ``` 118 | -------------------------------------------------------------------------------- /lib/module.js: -------------------------------------------------------------------------------- 1 | var request = require('request') 2 | , merge = require('lodash.merge') 3 | 4 | 5 | function leading(n, d) { 6 | d = d || 2 7 | n += '' 8 | while (n.length < d) n = "0" + n 9 | return n 10 | } 11 | 12 | function ymd(date) { 13 | date = new Date(date) 14 | return [date.getFullYear(), leading(date.getMonth() + 1), leading(date.getDate())].join('-') 15 | } 16 | 17 | module.exports = function(modules, downloadUrl, users, mainopts) { 18 | function Module(name) { 19 | if (!(this instanceof Module)) return new Module(name) 20 | this.name = name 21 | } 22 | 23 | Module.prototype.info = function(options, callback) { 24 | return modules.get(this.name, callback) 25 | } 26 | Module.prototype.version = function(options, callback) { 27 | return modules.get('_design/app/_show/package/' + this.name, { 28 | version: options.string 29 | }, callback) 30 | } 31 | 32 | function getDownloader(methodName, defs) { 33 | defs = merge({ 34 | detail: 'range', 35 | since: '2000-01-01', 36 | until: '3000-01-01' 37 | }, defs) 38 | function Downloader(options, callback) { 39 | var detail = options.detail || defs.detail 40 | , period = (options.since = options.since ? ymd(options.since) : defs.since) + ':' + 41 | (options.until = options.until ? ymd(options.until) : defs.until) 42 | var url = [downloadUrl, detail, period, this.name].join('/') 43 | return request.get({ url:url, strictSSL:false }, callback) 44 | } 45 | return Downloader 46 | } 47 | 48 | Module.prototype.downloads = getDownloader('downloads') 49 | Module.prototype.downloads.select = ['downloads', true] 50 | Module.prototype.downloads.map = function(row) { 51 | return { date:row.day, value:row.downloads } 52 | } 53 | 54 | Module.prototype.totalDownloads = getDownloader('totalDownloads') 55 | Module.prototype.totalDownloads.select = ['downloads', true] 56 | Module.prototype.totalDownloads.single = true 57 | Module.prototype.totalDownloads.reduce = function(acc, row) { 58 | return acc + (row.downloads || 0) 59 | } 60 | Module.prototype.totalDownloads.reduce.start = 0 61 | 62 | Module.prototype.stars = function(options, callback) { 63 | return modules.get('_design/app/_view/starredByPackage', { 64 | startkey: this.name 65 | , endkey: this.name 66 | }, callback) 67 | } 68 | Module.prototype.stars.select = ['rows', true, 'value'] 69 | 70 | Module.prototype.latest = function(options, callback) { 71 | return modules.get('_design/app/_view/byField', { 72 | key: this.name 73 | }, callback) 74 | } 75 | Module.prototype.latest.select = ['rows', true, 'value'] 76 | Module.prototype.latest.single = true 77 | 78 | Module.prototype.field = function(options, callback) { 79 | var field = options.field || options.string 80 | 81 | return modules.get('_design/app/_list/byField/byField', { 82 | field: field 83 | , key: this.name 84 | }, callback) 85 | } 86 | 87 | Module.prototype.size = function(options, callback) { 88 | return modules.get('_design/app/_view/howBigIsYourPackage', { 89 | key: this.name 90 | }, callback) 91 | } 92 | Module.prototype.size.select = ['rows', true, 'value'] 93 | Module.prototype.size.single = true 94 | 95 | Module.prototype.dependents = function(options, callback) { 96 | return modules.get('_design/app/_view/dependedUpon', { 97 | group_level: 2 98 | , startkey: [this.name] 99 | , endkey: [this.name, {}] 100 | }, callback) 101 | } 102 | Module.prototype.dependents.select = ['rows', true, 'key'] 103 | Module.prototype.dependents.map = function(row) { 104 | return row[1] 105 | } 106 | 107 | return Module 108 | } -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var es = require('event-stream') 2 | , reducestream = require('stream-reduce') 3 | , jsonstream = require('JSONStream') 4 | , merge = require('lodash.merge') 5 | 6 | function passthrough(data) { 7 | return data 8 | } 9 | 10 | module.exports = exports = stats 11 | 12 | // pristine copy of default options 13 | var DEFAULTS = { 14 | registry: 'https://skimdb.npmjs.com/', 15 | modules: 'registry', 16 | // https://api.npmjs.org/downloads/ :detail=(point|range) / :period=(last-month|last-week|last-day|YYYY-MM-DD:YYYY-MM-DD) / :package? 17 | downloads: 'https://api.npmjs.org/downloads', 18 | users: 'public_users', 19 | dirty: false 20 | } 21 | 22 | // inherit to ensure original defaults peek through if keys get deleted 23 | var GLOBAL_DEFAULTS = merge(Object.create(DEFAULTS), DEFAULTS) 24 | 25 | exports.defaults = function(defaults){ 26 | return merge(GLOBAL_DEFAULTS, defaults) 27 | } 28 | 29 | function stats(registry, mainopts) { 30 | if (typeof registry === 'object') { 31 | mainopts = registry 32 | registry = undefined 33 | } 34 | 35 | mainopts = merge({}, GLOBAL_DEFAULTS, mainopts, {registry:registry}); 36 | var nanoConf = merge({url: mainopts.registry}, mainopts.nano) 37 | 38 | var nano = require('nano')(nanoConf) 39 | , modules = nano.db.use(mainopts.modules) 40 | , users = nano.db.use(mainopts.users) 41 | , downloadUrl = mainopts.downloads 42 | 43 | var Keyword = require('./lib/keyword')(modules, downloadUrl, users, mainopts) 44 | , Module = require('./lib/module')(modules, downloadUrl, users, mainopts) 45 | , User = require('./lib/user')(modules, downloadUrl, users, mainopts) 46 | , Registry = require('./lib/registry')(modules, downloadUrl, users, mainopts) 47 | 48 | function modifier(method) { 49 | return function(options, callback) { 50 | options = options || {} 51 | 52 | if (typeof options === 'string') { 53 | options = { string: options } 54 | } 55 | if (typeof options === 'function') { 56 | callback = options 57 | options = {} 58 | } 59 | 60 | if (!method.select || mainopts.dirty) { 61 | return method.call(this, options, callback) 62 | } 63 | 64 | var buffer = '' 65 | var write = callback ? function write(data) { 66 | buffer += data 67 | this.queue(data) 68 | } : function(data) { 69 | this.queue(data) 70 | } 71 | 72 | var stream = es.pipeline( 73 | method.call(this, options) 74 | , jsonstream.parse(method.select) 75 | , es.mapSync(method.map || passthrough) 76 | , method.reduce 77 | ? reducestream(method.reduce, method.reduce.start) 78 | : es.mapSync(passthrough) 79 | , method.single 80 | ? es.stringify() 81 | : jsonstream.stringify('[', ',', ']') 82 | , es.through(write, end) 83 | ) 84 | 85 | if (callback) stream.on('error', callback) 86 | 87 | function end() { 88 | var self = this 89 | 90 | if (callback) { 91 | try { 92 | callback(null, JSON.parse(buffer)) 93 | } catch(e) { 94 | callback(e) 95 | } 96 | return 97 | } 98 | 99 | this.queue(null) 100 | } 101 | 102 | return stream 103 | } 104 | } 105 | 106 | ;[Registry, Keyword, User, Module].forEach(function(model) { 107 | Object.keys(model.prototype).forEach(function(name) { 108 | var method = model.prototype[name] 109 | 110 | model.prototype[name] = modifier(model.prototype[name]) 111 | }) 112 | }) 113 | 114 | var registry = new Registry 115 | 116 | registry.user = User 117 | registry.module = Module 118 | registry.keyword = Keyword 119 | 120 | return registry 121 | } 122 | --------------------------------------------------------------------------------