├── .gitignore ├── README.md ├── get.js ├── index.js ├── package.json ├── server.js └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .dat 3 | dat.json 4 | .DS_Store 5 | seq.json 6 | *.db 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dat-npm 2 | 3 | npm for [dat](https://github.com/maxogden/dat). work in progress 4 | 5 | stores data in this format: 6 | 7 | ``` 8 | { 9 | "versions": { 10 | "0.0.0": { 11 | "name": "ndarray-json", 12 | "version": "0.0.0", 13 | "dependencies": { 14 | "dtype": "~0.1.0", 15 | "ndarray": "~1.0.5", 16 | "cwise": "~0.4.0" 17 | }, 18 | "devDependencies": { 19 | "zeros": "0.0.0", 20 | "tape": "~1.0.4", 21 | "cave-automata-2d": "~0.3.1" 22 | }, 23 | "dist": { 24 | "shasum": "ebfd81ecb038d280610504e885d4c20b768a1dc5", 25 | "tarball": "http://registry.npmjs.org/ndarray-json/-/ndarray-json-0.0.0.tgz" 26 | } 27 | }, 28 | "0.1.0": { 29 | "name": "ndarray-json", 30 | "version": "0.1.0", 31 | "dependencies": { 32 | "ndarray": "~1.0.5", 33 | "tab64": "0.0.0" 34 | }, 35 | "devDependencies": { 36 | "zeros": "0.0.0", 37 | "tape": "~1.0.4", 38 | "cave-automata-2d": "~0.3.1" 39 | }, 40 | "dist": { 41 | "shasum": "ea992ab3ddc9e7fac5644ec0e44cbfb9e2672dcd", 42 | "tarball": "http://registry.npmjs.org/ndarray-json/-/ndarray-json-0.1.0.tgz" 43 | } 44 | }, 45 | "0.1.1": { 46 | "name": "ndarray-json", 47 | "version": "0.1.1", 48 | "dependencies": { 49 | "ndarray": "~1.0.5", 50 | "tab64": "0.0.1" 51 | }, 52 | "devDependencies": { 53 | "zeros": "0.0.0", 54 | "tape": "~1.0.4", 55 | "cave-automata-2d": "~0.3.1" 56 | }, 57 | "dist": { 58 | "shasum": "dd73606ef55188fbcc7e00c3a56c071f7a805ce1", 59 | "tarball": "http://registry.npmjs.org/ndarray-json/-/ndarray-json-0.1.1.tgz" 60 | } 61 | } 62 | }, 63 | "name": "ndarray-json", 64 | "dist-tags": { 65 | "latest": "0.1.1" 66 | }, 67 | "modified": "2013-10-19T04:12:06.558Z" 68 | } 69 | ``` -------------------------------------------------------------------------------- /get.js: -------------------------------------------------------------------------------- 1 | var hyperdrive = require('hyperdrive') 2 | var hyperdb = require('hyperdb') 3 | var hyperdiscovery = require('hyperdiscovery') 4 | 5 | var DatNPM = require('./') 6 | var keys = { 7 | meta: 'b87bbf4afad0ecb9c12ae6e14605251c938074de5dbda76ab45f740d90283dfa', 8 | tarballs: 'c1cd8b35e142fd25055e53a0c18ece5f91d4845c899faa8c3d1d297adf0ba68e' 9 | } 10 | 11 | var module = process.argv[2] || 'pushpop' 12 | 13 | DatNPM(keys, function (err, datNpm) { 14 | datNpm.meta.once('remote-update', function () { 15 | datNpm.meta.get('/modules/' + module, function (err, data) { 16 | if (err) return console.log(err) 17 | var meta = data[0].value 18 | console.log("metadata", JSON.stringify(meta)) 19 | var latest = module + '-' + meta['dist-tags'].latest + '.tgz' 20 | datNpm.tarballs.stat(DatNPM.hashFilename(latest), function (err, stat) { 21 | if (err) throw err 22 | console.log("stat", JSON.stringify(stat)) 23 | datNpm.tarballs.close() 24 | datNpm.tarballs.swarm.close() 25 | datNpm.meta.swarm.close() 26 | }) 27 | }) 28 | }) 29 | }) -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | var path = require('path') 3 | var crypto = require('crypto') 4 | var request = require('request') 5 | var through = require('through2') 6 | var pump = require('pump') 7 | var ndjson = require('ndjson') 8 | var concat = require('concat-stream') 9 | var parallel = require('parallel-transform') 10 | var hyperdb = require('hyperdb') 11 | var hyperdrive = require('hyperdrive') 12 | var hyperdiscovery = require('hyperdiscovery') 13 | var minify = require('minify-registry-metadata') 14 | var parallel = require('parallel-transform') 15 | 16 | var PARALLEL = 1024 17 | 18 | module.exports = function (keys, cb) { 19 | if (typeof keys === 'function') { 20 | cb = keys 21 | keys = null 22 | } 23 | var meta 24 | var tarballs = hyperdrive('./npm-tarballs.db', keys && keys.tarballs, {live: true}) 25 | tarballs.on('ready', function () { 26 | var tarballSwarm = hyperdiscovery(tarballs) 27 | meta = hyperdb('./npm-meta.db', keys && keys.meta, {sparse: true, valueEncoding: 'json'}) 28 | meta.on('ready', function () { 29 | var metaSwarm = hyperdiscovery(meta, {live: true}) 30 | meta.swarm = metaSwarm 31 | tarballs.swarm = tarballSwarm 32 | cb(null, {meta: meta, tarballs: tarballs, startUpdating: startUpdating}) 33 | }) 34 | }) 35 | 36 | function startUpdating (err) { 37 | if (err) { 38 | log('Error updating: %s - retrying in 60s', err.message) 39 | return setTimeout(startUpdating, 60000) 40 | } 41 | 42 | latestSeq(function (err, seq) { 43 | if (err) throw err 44 | 45 | seq = Math.max(0, seq - 1) // sub 1 incase of errors 46 | 47 | if (seq) log('Continuing fetching npm data from seq: %d', seq) 48 | 49 | var url = 'https://skimdb.npmjs.com/registry/_changes?heartbeat=30000&include_docs=true&feed=continuous' + (seq ? '&since=' + seq : '') 50 | 51 | pump(request(url), ndjson.parse(), save(), startUpdating) 52 | }) 53 | } 54 | 55 | function latestSeq (cb) { 56 | meta.get('/latest-seq', function (err, val) { 57 | if (err || !val) return cb(null, 0) 58 | var seq = val[0].value 59 | cb(null, seq) 60 | }) 61 | } 62 | 63 | function tick (fn, err, val) { 64 | process.nextTick(function () { 65 | fn(err, val) 66 | }) 67 | } 68 | 69 | function log (fmt) { 70 | fmt = '[dat-npm] ' + fmt 71 | console.error.apply(console, arguments) 72 | } 73 | 74 | function save () { 75 | return through.obj(function (data, enc, cb) { 76 | var doc = data.doc 77 | if (!doc) return cb() 78 | if (data.id.match(/^_design\//)) return cb() 79 | var key = doc._id 80 | if (doc._deleted) { 81 | // TODO 82 | return cb() 83 | } 84 | var metadata = minify(doc) 85 | var tarballs = Object.keys(metadata.versions).map(function (v) { 86 | var dist = metadata.versions[v].dist 87 | return { 88 | filename: path.basename(dist.tarball), 89 | url: dist.tarball 90 | } 91 | }) 92 | downloadTarballs(tarballs, function (err) { 93 | if (err) return cb(err) 94 | meta.put('/modules/' + key, metadata, function (err) { 95 | if (err) return cb(err) 96 | log('wrote /modules/' + key + ', seq=' + data.seq) 97 | meta.put('/latest-seq', data.seq, cb) 98 | }) 99 | }) 100 | }) 101 | } 102 | 103 | function downloadTarballs (items, done) { 104 | var transform = parallel(24, function (i, cb) { 105 | var filename = module.exports.hashFilename(i.filename) 106 | tarballs.stat(filename, function (err, stat) { 107 | if (stat) return cb() // already have it 108 | dlTarball() 109 | }) 110 | function dlTarball () { 111 | log('GET', i.url) 112 | var r = request(i.url) 113 | r.on('error', function (err) { 114 | log('Request error: ' + err.message + ' - ' + i.url) 115 | return cb(err) 116 | }) 117 | r.on('response', function (re) { 118 | if (re.statusCode === 404) { 119 | log('404 ' + i.url) 120 | return cb() // ignore 404s 121 | } 122 | if (re.statusCode === 503) { 123 | log('503 ' + i.url) 124 | return cb(null) // ignore forbidden 125 | } 126 | if (re.statusCode > 299) { 127 | return pump(re, concat(function (resp) { 128 | // https://github.com/npm/registry/issues/213 129 | if (resp.toString().match('Error fetching package from tmp remote')) { 130 | log('500 tmp remote error: ' + i.url) 131 | return cb(null) // ignore this error for now 132 | } 133 | if (resp.toString().match('InternalError')) { 134 | log('500 InternalError: ' + i.url) 135 | return cb(null) // ignore this error for now 136 | } 137 | return cb(new Error('Status: ' + re.statusCode + ' ' + i.url)) 138 | }), function (err) { 139 | if (err) console.log('concat error ' + err.message + ' ' + i.url) 140 | }) 141 | } 142 | var ws = tarballs.createWriteStream(filename) 143 | pump(re, ws, function (err) { 144 | if (err) { 145 | err.errType = 'streamPumpErr' 146 | return cb(err) 147 | } 148 | cb(null) 149 | }) 150 | }) 151 | } 152 | }) 153 | items.forEach(function (i) { transform.write(i) }) 154 | transform.end() 155 | var drain = concat(function (results) {}) // ignore results 156 | pump(transform, drain, done) 157 | } 158 | } 159 | 160 | // only needed until hyperdb lands in hyperdrive 161 | module.exports.hashFilename = function (filename) { 162 | var h = crypto.createHash('sha256').update(filename).digest('hex') 163 | return `${h.slice(0, 2)}/${h.slice(2, 4)}/${h.slice(4, 6)}/${h.slice(6, 8)}/${h.slice(8)}` 164 | } 165 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dat-npm", 3 | "version": "4.0.6", 4 | "description": "npm registry on dat", 5 | "main": "index.js", 6 | "dependencies": { 7 | "concat-stream": "^1.4.6", 8 | "hyperdb": "^1.1.0-rc1", 9 | "hyperdiscovery": "^6.0.4", 10 | "hyperdrive": "^9.5.1", 11 | "minify-registry-metadata": "^2.0.0", 12 | "ndjson": "^1.5.0", 13 | "parallel-transform": "^0.2.2", 14 | "pump": "^0.3.5", 15 | "request": "^2.40.0", 16 | "routes-router": "^4.3.0", 17 | "run-parallel": "^1.1.6", 18 | "semver": "^5.4.1", 19 | "through2": "^0.6.1", 20 | "validate-npm-package-name": "^3.0.0" 21 | }, 22 | "devDependencies": { 23 | "rimraf": "^2.6.1", 24 | "tape": "^4.8.0" 25 | }, 26 | "scripts": { 27 | "test": "node test.js" 28 | }, 29 | "repository": { 30 | "type": "git", 31 | "url": "git://github.com/mafintosh/dat-npm" 32 | }, 33 | "keywords": [ 34 | "dat", 35 | "npm" 36 | ], 37 | "author": "Mathias Buus", 38 | "license": "MIT", 39 | "bugs": { 40 | "url": "https://github.com/mafintosh/dat-npm/issues" 41 | }, 42 | "homepage": "https://github.com/mafintosh/dat-npm" 43 | } 44 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | var DatNPM = require('./index.js') 2 | DatNPM(function (err, datNpm) { 3 | console.log('Sharing hypercores', { 4 | tarballs: datNpm.tarballs.key.toString('hex'), 5 | meta: datNpm.meta.key.toString('hex') 6 | }) 7 | datNpm.startUpdating() 8 | }) 9 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var http = require('http') 2 | var test = require('tape') 3 | var request = require('request') 4 | var rimraf = require('rimraf') 5 | var restApi = require('./rest-api.js') 6 | 7 | rimraf.sync('./npm-meta.db') 8 | rimraf.sync('./npm-tarballs.db') 9 | 10 | test('get tarball', function (t) { 11 | var server = restApi(function (err, router, datNpm) { 12 | var server = http.createServer(function (req, res) { 13 | console.log(req.method, req.url) 14 | router(req, res) 15 | }) 16 | server.listen(8888, function () { 17 | request({json: true, url: 'http://localhost:8888/cache/request/2.81.0'}, function (err, resp, data) { 18 | t.equal(resp.statusCode, 200, '200 OK') 19 | var key = '/tarballs/request-2.81.0.tgz' 20 | t.deepEqual(data, {key: key, ready: true}, 'json matches') 21 | datNpm.tarballs.stat(key, function (err, stat) { 22 | t.ifErr(err, 'no error') 23 | t.ok(stat, 'stat ok') 24 | server.close(function () { 25 | datNpm.tarballs.close() 26 | datNpm.tarballs.swarm.close() 27 | datNpm.meta.swarm.close() 28 | t.end() 29 | }) 30 | }) 31 | }) 32 | }) 33 | }) 34 | }) 35 | --------------------------------------------------------------------------------