├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── index.js ├── lib └── import.js ├── package.json └── test ├── data ├── dummy.json └── sample.csv ├── import.js └── jawn.js /.gitignore: -------------------------------------------------------------------------------- 1 | tmp 2 | node_modules 3 | data.jawn 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "4.1" 4 | - "4.0" 5 | - "0.12" 6 | - "0.11" 7 | - "0.10" 8 | - "iojs" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Matt Zumwalt. All rights reserved. 2 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 3 | 4 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 5 | 6 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 7 | 8 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 9 | 10 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dat jawn: 'Git for Tabular Data' 2 | 3 | [![Build Status](https://travis-ci.org/CfABrigadePhiladelphia/jawn.svg?branch=master)](https://travis-ci.org/CfABrigadePhiladelphia/jawn) 4 | 5 | [![NPM](https://nodei.co/npm/jawn.png)](https://nodei.co/npm/jawn/) 6 | 7 | Jawn is a node.js module that allows _distributed version control of Tabular Data_. It's connected to the [dat](https://github.com/maxogden/dat) project. It allows you to import tabular data (rows and columns like CSV or TSV) and track how those data change over time. _Do you have non-tabular data? read this:_ [What about Non Tabular Data?](https://github.com/CfABrigadePhiladelphia/jawn/wiki/What-about-Non-Tabular-Data%3F) 8 | 9 | The key features for jawn are to: 10 | * **manage and track change history** in tabular data 11 | * **create historical checkpoints** with metadata (e.g., message, timestamp, author) 12 | 13 | Jawn relies on [hypercore](https://github.com/mafintosh/hypercore) to handle the core functions around creating merkle chains, which allows us to 14 | 15 | * **supply access points to data** across the network with a peer-to-peer model 16 | * **sync incrementally** between machines 17 | 18 | This is where jawn connects with the current work of the dat team, who created hypercore and are using it to do the same things with directories of files. For more background info, read our [Technical Background and Reference Code Bases](https://github.com/CfABrigadePhiladelphia/jawn/wiki/Technical-Background-and-Reference-Code-Bases) wiki page. 19 | 20 | ## Project Team 21 | 22 | jawn is maintained by a [Code for Philly](https://codeforphilly.org) project that aims to be a model for mentorship and collaborative learning. For full information about the project go to the [jawn project page](https://codeforphilly.org/projects/dat_tables) 23 | 24 | We welcome contributions from anyone. 25 | 26 | ## Usage 27 | 28 | _work in progress_ 29 | 30 | ## Contributing 31 | 32 | _work in progress_ 33 | 34 | If you want to work on the jawn code, first clone the repository and cd into that directory, then install the node dependencies and run the tests with these commands: 35 | 36 | ``` 37 | npm install 38 | npm test 39 | ``` 40 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var level = require('level') 2 | var hypercore = require('hypercore') 3 | var createImportPipeline = require('./lib/import.js') 4 | 5 | module.exports = Jawn 6 | 7 | function Jawn (opts) { 8 | if (!opts) opts = {} 9 | this.core = initializeHypercore(opts) 10 | this.db = this.core.db 11 | } 12 | 13 | Jawn.prototype.createImportPipeline = function (opts) { 14 | return createImportPipeline(this, opts) 15 | } 16 | 17 | // Initializes hypercore and its database 18 | // @default Creates a leveldb database called `data.jawn` and initializes hypercore using that db 19 | // @option 'core' the hypercore instance to use 20 | // @option 'db' the db instace (leveldb) to initialize hypercore with. This is ignored if you use the `core` option 21 | function initializeHypercore (opts) { 22 | var core 23 | if (opts.hasOwnProperty('core')) { 24 | core = opts.core 25 | } else { 26 | var db = opts.db || level('data.jawn') 27 | core = hypercore(db) 28 | } 29 | return core 30 | } 31 | -------------------------------------------------------------------------------- /lib/import.js: -------------------------------------------------------------------------------- 1 | var miss = require('mississippi') 2 | var through = require('through2') 3 | var parseInputStream = require('parse-input-stream') 4 | 5 | module.exports = importPipeline 6 | 7 | // Returns the parser at the beginning of the pipeline 8 | // When the pipeline finishes, it calls `callback(feedId, err)`. 9 | // If there were no errors, err will be undefined. 10 | function importPipeline (jawn, opts) { 11 | if (!opts) opts = {} 12 | var writeStream = jawn.core.createWriteStream(opts) 13 | var parser = parseInputStream(opts) 14 | var transform = through.obj(stringifyData, end) 15 | 16 | var importStream = miss.pipeline(parser, transform, writeStream) 17 | 18 | function stringifyData (data, enc, next) { 19 | this.push(JSON.stringify(data)) 20 | next() 21 | } 22 | 23 | function end (done) { 24 | done() 25 | } 26 | 27 | importStream.writeStream = writeStream 28 | 29 | return importStream 30 | } 31 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jawn", 3 | "version": "0.0.1", 4 | "description": "Tabular Data support for Dat/hypercore", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "standard && tape test/*.js" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/CfABrigadePhiladelphia/jawn.git" 12 | }, 13 | "keywords": [ 14 | "data", 15 | "version control" 16 | ], 17 | "author": "Matt Zumwalt", 18 | "license": "MIT", 19 | "bugs": { 20 | "url": "https://github.com/CfABrigadePhiladelphia/jawn/issues" 21 | }, 22 | "homepage": "https://github.com/CfABrigadePhiladelphia/jawn", 23 | "dependencies": { 24 | "fs": "0.0.2", 25 | "hypercore": "~1.8.0", 26 | "level": "~1.4.0", 27 | "mississippi": "^1.2.0", 28 | "parse-input-stream": "^1.0.1", 29 | "path": "~0.12.7", 30 | "tape": "~4.5.1", 31 | "through2": "^2.0.1" 32 | }, 33 | "devDependencies": { 34 | "standard": "^6.0.5", 35 | "tape": "^4.4.0", 36 | "memdb": "^1.3.1" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /test/data/dummy.json: -------------------------------------------------------------------------------- 1 | {"foo":"bar","name":"josie","age":"35"} 2 | {"foo":"baz","name":"eloise","age":"71"} 3 | {"foo":"baz","name":"francoise","age":"5"} 4 | -------------------------------------------------------------------------------- /test/data/sample.csv: -------------------------------------------------------------------------------- 1 | Type of Experience,Little/No Experience,Some Experience,Very Familiar 2 | Writing software in any programming language,1,5,4 3 | Frontend Web Development,4,3,3 4 | Server-side (“backend”) Web Development,4,4,2 5 | "Using Git to track changes and share code (add, commit, push, pull)",2,5,3 -------------------------------------------------------------------------------- /test/import.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var fs = require('fs') 3 | var path = require('path') 4 | var Jawn = require('../') 5 | var memdb = require('memdb') 6 | 7 | test('import json to jawn', function (t) { 8 | var jawn = freshJawn() 9 | var importStream = importFromFile(jawn, 'dummy.json', {'format': 'json'}) 10 | var expected = [ 11 | '{"foo":"bar","name":"josie","age":"35"}', 12 | '{"foo":"baz","name":"eloise","age":"71"}', 13 | '{"foo":"baz","name":"francoise","age":"5"}' 14 | ] 15 | 16 | importStream.on('finish', verify) 17 | 18 | function verify (err) { 19 | if (err) { console.log(err) } 20 | var feedId = importStream.writeStream.id 21 | var rs = jawn.core.createReadStream(feedId) 22 | rs.on('data', function (block) { 23 | t.same(block.toString(), expected.shift(), 'block matches imported line') 24 | }) 25 | t.same(jawn.core.get(feedId).blocks, 3, 'correct number of blocks returned') 26 | t.end() 27 | } 28 | }) 29 | 30 | test('import csv to jawn', function (t) { 31 | var jawn = freshJawn() 32 | importFromFile(jawn, 'sample.csv', {'format': 'csv'}) 33 | var importStream = importFromFile(jawn, 'sample.csv', {'format': 'csv'}, verify) 34 | var expected = [ 35 | '{"Type of Experience":"Writing software in any programming language","Little/No Experience":"1","Some Experience":"5","Very Familiar":"4"}', 36 | '{"Type of Experience":"Frontend Web Development","Little/No Experience":"4","Some Experience":"3","Very Familiar":"3"}', 37 | '{"Type of Experience":"Server-side (“backend”) Web Development","Little/No Experience":"4","Some Experience":"4","Very Familiar":"2"}', 38 | '{"Type of Experience":"Using Git to track changes and share code (add, commit, push, pull)","Little/No Experience":"2","Some Experience":"5","Very Familiar":"3"}' 39 | ] 40 | 41 | importStream.on('finish', verify) 42 | 43 | function verify (err) { 44 | if (err) { console.log(err) } 45 | var feedId = importStream.writeStream.id 46 | var rs = jawn.core.createReadStream(feedId) 47 | rs.on('data', function (block) { 48 | t.same(block.toString(), expected.shift(), 'block matches imported line') 49 | }) 50 | t.same(jawn.core.get(feedId).blocks, 4, 'correct number of blocks returned') 51 | t.end() 52 | } 53 | }) 54 | 55 | // helpers 56 | 57 | function fixture (name) { 58 | return path.join(__dirname, 'data', name) 59 | } 60 | 61 | function freshJawn () { 62 | return new Jawn({db: memdb()}) 63 | } 64 | 65 | function importFromFile (jawn, file, opts) { 66 | var importPipeline = jawn.createImportPipeline(opts) 67 | var data = fs.createReadStream(fixture(file)) 68 | data.pipe(importPipeline) 69 | return importPipeline 70 | } 71 | -------------------------------------------------------------------------------- /test/jawn.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var Jawn = require('../index') 3 | var memdb = require('memdb') 4 | var hypercore = require('hypercore') 5 | 6 | test('jawn constructor defaults', function (t) { 7 | var jawn = new Jawn({db: memdb()}) 8 | t.true(jawn.core instanceof hypercore, 'core is an instance of Hypercore') 9 | // t.true(jawn.db instanceof leveldb, 'db is an instance of leveldb') 10 | t.equal(jawn.db, jawn.core.db, 'jawn db matches core db') 11 | t.end() 12 | }) 13 | 14 | test('jawn constructor set db', function (t) { 15 | var sampledb = memdb() 16 | var jawn = new Jawn({db: sampledb}) 17 | t.equal(jawn.core.db, sampledb, 'initializes hypercore with the provided db') 18 | t.end() 19 | }) 20 | 21 | test('jawn constructor set core', function (t) { 22 | var samplecore = hypercore(memdb()) 23 | var jawn = new Jawn({core: samplecore}) 24 | t.equal(jawn.core, samplecore, 'uses the provided instance of hypercore') 25 | t.end() 26 | }) 27 | --------------------------------------------------------------------------------