├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── cli.js ├── index.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 'node' 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016-2017 Thomas Watson Steen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # npm-to-hypercore 2 | 3 | A simple data fetcher that streams all changes made to npm and stores 4 | them in Hypercore. The changes are stored such that each version of a 5 | package that's released is one block in Hypercore. 6 | 7 | [![Build status](https://travis-ci.org/watson/npm-to-hypercore.svg?branch=master)](https://travis-ci.org/watson/npm-to-hypercore) 8 | [![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](https://github.com/feross/standard) 9 | 10 | ## Installation 11 | 12 | ``` 13 | npm install npm-to-hypercore -g 14 | ``` 15 | 16 | ## Usage 17 | 18 | ``` 19 | npm-to-hypercore [db] 20 | ``` 21 | 22 | Just run the `npm-to-hypercore` command. Takes an optional `db` argument 23 | to use as the path to the database. 24 | 25 | ## Try it out 26 | 27 | We have an instance of this running already. You can replicate data from it by using this hypercore key 28 | 29 | ``` 30 | f5d045813912dadbff4bdc8a43bb78da6685f965f1a88e430db49c793a5a1a01 31 | ``` 32 | 33 | To test it out you can use hypertail 34 | 35 | ``` 36 | hypertail /tmp/npm-to-hypercore f5d045813912dadbff4bdc8a43bb78da6685f965f1a88e430db49c793a5a1a01 37 | ``` 38 | 39 | ## License 40 | 41 | MIT 42 | -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use strict' 3 | 4 | require('./') 5 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var path = require('path') 4 | var mkdirp = require('mkdirp') 5 | var ChangesStream = require('changes-stream') 6 | var clean = require('normalize-registry-metadata') 7 | var through2 = require('through2') 8 | var hypercore = require('hypercore') 9 | var swarm = require('hyperdrive-archive-swarm') 10 | var pump = require('pump') 11 | var level = require('level') 12 | 13 | var dbRoot = process.argv[2] || path.join('.', 'npm-to-hypercore.db') 14 | console.log('db location: %s', path.resolve(dbRoot)) 15 | mkdirp(dbRoot) 16 | 17 | var feed = hypercore(path.join(dbRoot, 'hypercore'), {valueEncoding: 'json'}) 18 | var db = level(path.join(dbRoot, 'index')) 19 | var normalize = through2.obj(transform) 20 | 21 | feed.ready(function () { 22 | console.log('hypercore key:', feed.key.toString('hex')) 23 | swarm(feed) 24 | run() 25 | }) 26 | 27 | function run (err) { 28 | if (err) throw err 29 | recoverFromBadShutdown(function (err) { 30 | if (err) throw err 31 | getNextSeqNo(function (err, seq) { 32 | if (err) throw err 33 | console.log('feching changes since sequence #%s', seq) 34 | pump(changesSinceStream(seq), normalize, run) 35 | }) 36 | }) 37 | } 38 | 39 | function changesSinceStream (seq) { 40 | return new ChangesStream({ 41 | db: 'https://replicate.npmjs.com', 42 | include_docs: true, 43 | since: seq, 44 | highWaterMark: 4 // reduce memory - default is 16 45 | }) 46 | } 47 | 48 | function transform (change, env, cb) { 49 | var doc = change.doc 50 | 51 | clean(doc) 52 | 53 | var modified = doc && doc.time && doc.time.modified 54 | var seq = change.seq 55 | 56 | if (!doc) { 57 | console.log('skipping %s - invalid document (seq: %s)', change.id, seq) 58 | done() 59 | return 60 | } else if (!doc.versions || doc.versions.length === 0) { 61 | console.log('skipping %s - no versions detected (seq: %s, modified: %s)', change.id, seq, modified) 62 | done() 63 | return 64 | } 65 | 66 | var versions = Object.keys(doc.versions) 67 | processVersion() 68 | 69 | function done (err) { 70 | if (err) return cb(err) 71 | db.put('!latest_seq!', change.seq, cb) 72 | } 73 | 74 | function processVersion (err) { 75 | if (err) return done(err) 76 | var version = versions.shift() 77 | if (!version) return done() 78 | var key = change.id + '@' + version 79 | db.get(key, function (err) { 80 | if (!err || !err.notFound) return processVersion(err) 81 | feed.append(doc.versions[version], function (err) { 82 | if (err) return done(err) 83 | if (feed.length % 1000 === 0) console.log('appended %d blocks (seq: %s, modified: %s)', feed.length, seq, modified) 84 | db.put(key, true, processVersion) 85 | }) 86 | }) 87 | } 88 | } 89 | 90 | function getNextSeqNo (cb) { 91 | db.get('!latest_seq!', function (err, seq) { 92 | if (err && err.notFound) cb(null, 0) 93 | else if (err) cb(err) 94 | else cb(null, parseInt(seq, 10) + 1) 95 | }) 96 | } 97 | 98 | // This function expects that there's no gab in the index. So if it 99 | // successfully locates key in the index there must no missing keys prior to 100 | // that. 101 | function recoverFromBadShutdown (cb) { 102 | console.log('validating index...') 103 | recover() 104 | 105 | function recover (index) { 106 | if (index === undefined) index = feed.length - 1 107 | if (index === -1) return done() 108 | feed.get(index, function (err, pkg) { 109 | if (err) return done(err) 110 | var key = pkg.name + '@' + pkg.version 111 | db.get(key, function (err) { 112 | if (!err || !err.notFound) return done(err) 113 | console.log('warning: block %d (%s) not indexed - recovering...', index, key) 114 | db.put(key, true, function (err) { 115 | if (err) return done(err) 116 | recover(index - 1) 117 | }) 118 | }) 119 | }) 120 | } 121 | 122 | function done (err) { 123 | if (err) return cb(err) 124 | console.log('index is up to date') 125 | cb() 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "npm-to-hypercore", 3 | "version": "3.0.2", 4 | "description": "Stream all of npm metadata into hypercore", 5 | "bin": "cli.js", 6 | "dependencies": { 7 | "changes-stream": "^2.2.0", 8 | "hypercore": "^5.4.2", 9 | "hyperdrive-archive-swarm": "^5.0.5", 10 | "level": "^1.6.0", 11 | "mkdirp": "^0.5.1", 12 | "normalize-registry-metadata": "^1.1.2", 13 | "pump": "^1.0.2", 14 | "through2": "^2.0.3" 15 | }, 16 | "devDependencies": { 17 | "standard": "^9.0.0" 18 | }, 19 | "scripts": { 20 | "test": "standard", 21 | "start": "node index.js" 22 | }, 23 | "author": "Thomas Watson Steen (https://twitter.com/wa7son)", 24 | "license": "MIT", 25 | "repository": { 26 | "type": "git", 27 | "url": "git+https://github.com/watson/npm-to-hypercore.git" 28 | }, 29 | "bugs": { 30 | "url": "https://github.com/watson/npm-to-hypercore/issues" 31 | }, 32 | "homepage": "https://github.com/watson/npm-to-hypercore#readme", 33 | "coordinates": [ 34 | 55.77757368361892, 35 | 12.589960523278819 36 | ] 37 | } 38 | --------------------------------------------------------------------------------