├── .gitignore ├── LICENSE ├── README.md ├── cli.js ├── index.html ├── index.js ├── package.json └── router.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # node-waf configuration 20 | .lock-wscript 21 | 22 | # Compiled binary addons (http://nodejs.org/api/addons.html) 23 | build/Release 24 | 25 | # Dependency directory 26 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 27 | node_modules 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Karissa McKelvey 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of dat-rest-server nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dat-rest-server 2 | 3 | A http server for [dat](http://dat-data.com). Parameters to routes are the same as the [cli documentation](https://github.com/maxogden/dat/blob/master/docs/cli-docs.md). 4 | 5 | ``` 6 | npm install -g dat-rest-server 7 | dat-rest-server --port= --path= 8 | ``` 9 | ## GET / 10 | 11 | Gets basic information about the state of the dat 12 | 13 | ``` 14 | { 15 | dat: true, 16 | version: "7.0.4", 17 | status: { 18 | head: "a2c77bedfb59f2237825614b57109ef35e097f5b1bf9320e9a52ef90768d566a", 19 | transaction: false, 20 | checkout: false, 21 | modified: "2015-09-01T22:53:17.775Z", 22 | datasets: 3, 23 | rows: 2686, 24 | files: 2, 25 | versions: 4, 26 | size: 1704743 27 | }, 28 | datasets: [ 29 | "organisms2", 30 | "organisms", 31 | "files" 32 | ] 33 | } 34 | ``` 35 | 36 | ## POST / 37 | 38 | Used for replication during clone: 39 | 40 | ``` 41 | dat clone mydatserver.com 42 | ``` 43 | 44 | ## GET /datasets 45 | 46 | ``` 47 | GET mydatserver.com/datasets 48 | ``` 49 | 50 | Returns 51 | ``` 52 | [ 53 | "files", 54 | "sac-crime", 55 | "models" 56 | ] 57 | ``` 58 | 59 | ## GET /changes 60 | 61 | Gets changes from dat 62 | 63 | Optional params: `live` 64 | 65 | ``` 66 | GET mydatserver.com/changes?live=true 67 | ``` 68 | 69 | Returns 70 | 71 | ``` 72 | {"root":true,"change":1,"date":"2015-09-01T20:47:40.582Z","version":"3ffbf6b75e57a934024c47e747390732bb0b8210310b037bb4974b5529820ea9","message":"","links":[],"puts":0,"deletes":0,"files":0} 73 | {"root":false,"change":2,"date":"2015-09-01T20:47:40.590Z","version":"dbf404f13567d5dd727534b80816b1b95211523611410024bafb53a3831f0f18","message":"","links":["3ffbf6b75e57a934024c47e747390732bb0b8210310b037bb4974b5529820ea9"],"puts":1,"deletes":0,"files":1} 74 | {"root":false,"change":3,"date":"2015-09-01T20:47:45.274Z","version":"969be9bd7e4670633e1b8412f9617311c80b6e3aec6d6013de15b705292fcb28","message":"","links":["dbf404f13567d5dd727534b80816b1b95211523611410024bafb53a3831f0f18"],"puts":1342,"deletes":0,"files":0} 75 | {"root":false,"change":4,"date":"2015-09-01T22:52:45.479Z","version":"2e8cd3a9578e9a5281768d8a7625adc2c7e99d60660fde5a8807205240d89601","message":"","links":["969be9bd7e4670633e1b8412f9617311c80b6e3aec6d6013de15b705292fcb28"],"puts":1,"deletes":0,"files":1} 76 | {"root":false,"change":5,"date":"2015-09-01T22:53:17.775Z","version":"a2c77bedfb59f2237825614b57109ef35e097f5b1bf9320e9a52ef90768d566a","message":"complete->completed","links":["2e8cd3a9578e9a5281768d8a7625adc2c7e99d60660fde5a8807205240d89601"],"puts":1342,"deletes":0,"files":0} 77 | ``` 78 | 79 | ## GET /files 80 | 81 | Get file list from dat. 82 | 83 | ``` 84 | GET mydatserver.com/files 85 | ``` 86 | 87 | ``` 88 | { 89 | files: [ 90 | "/Users/karissa/Dropbox/throwaway/bug.mov", 91 | "asdf", 92 | "black_crappie.json", 93 | "bug.mov", 94 | "fishing/black_crappie.json", 95 | "fishing/yellow_perch.json", 96 | "package.json" 97 | ] 98 | } 99 | ``` 100 | 101 | ## GET /files/:name 102 | 103 | Get file contents with the given name 104 | 105 | ``` 106 | GET mydatserver.com/files/package.json 107 | ``` 108 | 109 | ``` 110 | { 111 | name: "Hunting-and-Fishing", 112 | publisher: "waldoj", 113 | description: "Hunting-and-Fishing" 114 | } 115 | ``` 116 | 117 | ## GET /datasets/:name 118 | 119 | Gets dat data out. 120 | 121 | Optional params: `format` 122 | 123 | ``` 124 | GET mydatserver.com/datasets/?format=csv 125 | ``` 126 | 127 | Returns 128 | ``` 129 | cdatetime,address,district,beat,grid,crimedescr,ucr_ncic_code,latitude,longitude 130 | 1/1/06 0:00,3108 OCCIDENTAL DR,3,3C ,1115,10851(A)VC TAKE VEH W/O OWNER,2404,38.55042047,-121.3914158 131 | 1/1/06 0:00,2082 EXPEDITION WAY,5,5A ,1512,459 PC BURGLARY RESIDENCE,2204,38.47350069,-121.4901858 132 | 1/1/06 0:00,4 PALEN CT,2,2A ,212,10851(A)VC TAKE VEH W/O OWNER,2404,38.65784584,-121.4621009 133 | 1/1/06 0:00,22 BECKFORD CT,6,6C ,1443,476 PC PASS FICTICIOUS CHECK,2501,38.50677377,-121.4269508 134 | 1/1/06 0:00,3421 AUBURN BLVD,2,2A ,508,459 PC BURGLARY-UNSPECIFIED,2299,38.6374478,-121.3846125 135 | 1/1/06 0:00,5301 BONNIEMAE WAY,6,6B ,1084,530.5 PC USE PERSONAL ID INFO,2604,38.52697863,-121.4513383 136 | 1/1/06 0:00,2217 16TH AVE,4,4A ,957,459 PC BURGLARY VEHICLE,2299,38.537173,-121.4875774 137 | 1/1/06 0:00,3547 P ST,3,3C ,853,484 PC PETTY THEFT/INSIDE,2308,38.56433456,-121.4618826 138 | 1/1/06 0:00,3421 AUBURN BLVD,2,2A ,508,459 PC BURGLARY BUSINESS,2203,38.6374478,-121.3846125 139 | 1/1/06 0:00,1326 HELMSMAN WAY,1,1B ,444,1708 US THEFT OF MAIL,2310,38.60960217,-121.4918375 140 | ``` 141 | 142 | 143 | ## POST /datasets/:dataset 144 | 145 | Put data in dat. 146 | 147 | Optional params: `message`, `format`, `key`. 148 | 149 | ``` 150 | POST mydatserver.com/datasets/ 151 | ``` 152 | 153 | Be sure to push data in the request with an appropriate `Content-Type` header: 154 | 155 | ``` 156 | 'json': 'application/json', 157 | 'csv': 'text/csv', 158 | 'ndjson': 'application/x-ndjson', 159 | 'sse': 'text/event-stream' 160 | ``` 161 | 162 | -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | var args = require('minimist')(process.argv.slice(2)) 3 | var server = require('./') 4 | 5 | server(args, function (server, port) { 6 | server.listen(port) 7 | }) 8 | 9 | function usage () { 10 | console.log('dat-rest-server --port= --path=') 11 | } 12 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | hello -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var openDat = require('dat/lib/util/open-dat.js') 2 | var http = require('http') 3 | var getport = require('getport') 4 | 5 | var createRouter = require('./router.js') 6 | 7 | module.exports = function (args, cb) { 8 | if (!args.path) args.path = process.cwd() 9 | if (args.port) return serve(parseInt(args.port, 10)) 10 | 11 | getport(6442, function (err, port) { 12 | if (err) throw err 13 | return serve(port) 14 | }) 15 | 16 | function serve (port) { 17 | var router = createRouter() 18 | 19 | openDat(args, function (err, db) { 20 | if (err) return cb(err) 21 | if (!port) return cb(new Error('Invalid port: ' + port)) 22 | 23 | console.log('Listening on port ' + port) 24 | 25 | var server = http.createServer(function (req, res) { 26 | try { 27 | router(req, res, {db: db}, onError) 28 | } catch (err) { 29 | onError(err) 30 | } 31 | 32 | function onError (err) { 33 | console.trace(err) 34 | res.statusCode = err.statusCode || 500; 35 | res.end(err.message); 36 | } 37 | }) 38 | 39 | server.on('connection', function (socket) { 40 | socket.setNoDelay(true) // http://neophob.com/2013/09/rpc-calls-and-mysterious-40ms-delay/ 41 | }) 42 | 43 | // keep connections open for a long time (for e.g. --live) 44 | server.setTimeout(86400) 45 | 46 | cb(server, port) 47 | }) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dat-rest-server", 3 | "version": "1.0.6", 4 | "description": "An http server for [dat](http://dat-data.com).", 5 | "main": "index.js", 6 | "bin": { 7 | "dat-rest-server": "cli.js" 8 | }, 9 | "scripts": { 10 | "test": "echo \"Error: no test specified\" && exit 1" 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "https://github.com/karissa/dat-rest-server.git" 15 | }, 16 | "author": "Karissa McKelvey (http://karissamck.com/)", 17 | "license": "ISC", 18 | "bugs": { 19 | "url": "https://github.com/karissa/dat-rest-server/issues" 20 | }, 21 | "homepage": "https://github.com/karissa/dat-rest-server", 22 | "dependencies": { 23 | "dat": "^7.0.2", 24 | "debug": "^2.2.0", 25 | "getport": "^0.1.0", 26 | "http-hash-router": "^1.1.0", 27 | "minimist": "^1.1.2", 28 | "ndjson": "^1.4.1", 29 | "pump": "^1.0.0" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /router.js: -------------------------------------------------------------------------------- 1 | var Router = require('http-hash-router') 2 | var fs = require('fs') 3 | var path = require('path') 4 | var debug = require('debug')('dat-rest-server') 5 | var pump = require('pump') 6 | var ndjson = require('ndjson') 7 | var url = require('url') 8 | var createExportStream = require('dat/lib/export.js') 9 | var createImportStream = require('dat/lib/import.js') 10 | var listFilesStream = require('dat/lib/files.js') 11 | var route = require('dat/lib/serve.js') 12 | 13 | module.exports = function createRouter () { 14 | var router = Router() 15 | 16 | router.set('/', function (req, res, opts, cb) { 17 | if (req.method === 'GET') { 18 | route.information(opts.db, opts, function (err, str) { 19 | if (err) return cb(err) 20 | res.setHeader('content-type', 'application/json') 21 | res.end(str) 22 | }) 23 | } else if (req.method === 'POST') { 24 | return pump(req, route.replication(opts.db, opts), res) 25 | } else { 26 | res.statuscode = 405 27 | res.end() 28 | } 29 | }) 30 | 31 | router.set('/health', function (req, res, opts, cb) { 32 | var response = fs.readFileSync(path.join(__dirname, 'index.html')).toString() 33 | res.end(response) 34 | }) 35 | 36 | router.set('/changes', function (req, res, opts, cb) { 37 | var args = argparse(req) 38 | if (args.values === undefined) args.values = true 39 | return pump(opts.db.createChangesStream(args), ndjson.serialize(), res) 40 | }) 41 | 42 | router.set('/files', function (req, res, opts, cb) { 43 | var limit = opts.limit 44 | if (limit) { 45 | if (opts.limit) opts.limit = parseInt(limit, 10) 46 | } 47 | opts.dataset = opts.dataset || 'files' 48 | opts.json = true 49 | 50 | var filesStream = listFilesStream(opts.db, opts) 51 | return pump(filesStream, res, function (err) { 52 | if (err) throw err 53 | }) 54 | }) 55 | 56 | router.set('/files/:key', function (req, res, opts, cb) { 57 | opts.dataset = opts.dataset || 'files' 58 | if (req.method === 'GET') { 59 | var reader = opts.db.createFileReadStream(opts.params.key, opts) 60 | pump(reader, res) 61 | } else res.end('POST not supported yet.') 62 | }) 63 | 64 | router.set('/datasets', function (req, res, opts, cb) { 65 | opts.db.listDatasets(function (err, datasets) { 66 | if (err) return cb(err) 67 | res.end(JSON.stringify(datasets)) 68 | }) 69 | }) 70 | 71 | router.set('/datasets/:dataset', function (req, res, opts, cb) { 72 | var args = argparse(req) 73 | args.dataset = opts.params.dataset 74 | if (!args.format) args.format = 'ndjson' 75 | 76 | if (req.method === 'GET') return pump(createExportStream(opts.db, args), res) 77 | else return pump(req, createImportStream(opts.db, args), function done (err) { 78 | if (err) return cb(err) 79 | res.end(opts.db.head) 80 | }) 81 | }) 82 | 83 | return router 84 | } 85 | 86 | function argparse (req) { 87 | var parsedUrl = url.parse(req.url, true); 88 | return parsedUrl.query 89 | } 90 | --------------------------------------------------------------------------------