├── .gitignore ├── src ├── index.js ├── config.js └── gist-db.js ├── LICENSE ├── CHANGELOG.md ├── package.json ├── demo └── demo.js └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/* 2 | build/* 3 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | // This file is needed for commonJS require 2 | 3 | import gistDb from './gist-db' 4 | module.exports = gistDb 5 | -------------------------------------------------------------------------------- /src/config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | refreshMin: 10, 3 | github: { 4 | per_page: 100, 5 | timeout: 5000, 6 | version: '3.0.0' 7 | }, 8 | local: { 9 | save: 'NEVER', // NEVER, ON_REFRESH, ALWAYS 10 | location: null // ex. 'relative/path/to/local-storage.json' 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2013-2016 Matthew Chase Whittemore 4 | Copyright (c) 2016 pinn3 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 0.1.7 2 | 3 | * Bumped `request` dependency version due to security issues 4 | 5 | # 0.1.6 6 | 7 | * Fix initDb to return empty taffyDb if `config.local.save` was set to `NEVER` 8 | 9 | # 0.1.5 10 | 11 | * Added local storage functionality 12 | 13 | # 0.1.4 14 | 15 | * Request was being requried before each use, which was causing a timeout 16 | problems. Updated the code to use a single request object. 17 | 18 | # 0.1.3 19 | 20 | * Added userFileSave function, which allows users to do custom (async) actions 21 | to a file after the raw data has been pulled in. 22 | 23 | # 0.1.2 24 | 25 | * Added github authentication and thus the ability to access private gists 26 | 27 | # 0.1.1 28 | 29 | * Added gist object to file for meta data on the gist the file is from. 30 | 31 | * Changed database refresh to use merge rather than insert so items won't 32 | duplicate 33 | 34 | * Added check just before getRawFile and the add to database that checks if 35 | the file is in the db and if it is compares if the gist.updated_at of the 36 | new file is newer than that of the old file. This was can lower the number 37 | of calls to github and speed up the code a bit. 38 | 39 | * Changed github module to be my fork which supports since on the gist 40 | endpoints. Will change back once a new version of node-github is in NPM 41 | 42 | * Added since param to github calls, so we will only return gists added/edited 43 | since our last call. 44 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gist-db", 3 | "version": "0.1.7", 4 | "author": "pinn3 ", 5 | "description": "Treat your gist account like a database", 6 | "files": [ 7 | "build", 8 | "README.md" 9 | ], 10 | "contributors": [ 11 | { 12 | "name": "pinn3", 13 | "email": "iam@lacking.education" 14 | } 15 | ], 16 | "keywords": [ 17 | "database", 18 | "db", 19 | "gist", 20 | "github" 21 | ], 22 | "main": "./build", 23 | "scripts": { 24 | "prepublish": "npm test && npm run build", 25 | "postinstall": "npm run build", 26 | "prebuild": "rm -rf build && mkdir build", 27 | "build": "babel src -d build", 28 | "watch": "babel src -w -d build", 29 | "test": "standard", 30 | "demo": "npm install && node demo/demo.js" 31 | }, 32 | "standard": { 33 | "parser": "babel-eslint" 34 | }, 35 | "babel": { 36 | "presets": [ 37 | "es2015", 38 | "stage-2" 39 | ] 40 | }, 41 | "repository": { 42 | "type": "git", 43 | "url": "https://github.com/pinn3/gist-db.git" 44 | }, 45 | "dependencies": { 46 | "github": "https://github.com/mcwhittemore/node-github/tarball/master", 47 | "request": "2.68.0", 48 | "taffydb": "2.7.2" 49 | }, 50 | "license": "MIT", 51 | "devDependencies": { 52 | "babel-cli": "^6.9.0", 53 | "babel-eslint": "^6.0.4", 54 | "babel-preset-es2015": "^6.9.0", 55 | "babel-preset-stage-2": "^6.5.0", 56 | "standard": "^7.1.2" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /demo/demo.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const config = { 4 | github: { 5 | username: 'mcwhittemore' 6 | } 7 | } 8 | 9 | const fileInit = (file) => { 10 | // init custom group object 11 | file.groups = {} 12 | 13 | // define groups and their regex include rules 14 | const groupRules = { 15 | 'blog': /^Blog_/, 16 | 'project': /^Project_/, 17 | 'icon': /^Icon_$/, 18 | 'about': /^BlogAboutPage.md$/ 19 | } 20 | 21 | // get group names 22 | const groups = Object.keys(groupRules) 23 | 24 | // set file as excluded as we only want to include it if it has a group 25 | let include = false 26 | 27 | for (const group of groups) { 28 | const rule = groupRules[group] 29 | 30 | // check if filename matches regex rule 31 | if (file.filename.search(rule) > -1) { 32 | file.groups[group] = true // set included in group as true 33 | include = true // set include file as true 34 | } else { 35 | file.groups[group] = false 36 | } 37 | } 38 | 39 | if (include) { 40 | return file 41 | } else { 42 | return undefined 43 | } 44 | } 45 | 46 | const fileSave = (file, callback) => { 47 | file.html = file.raw 48 | callback(file) 49 | } 50 | 51 | const db = require('../build')(config, fileInit, fileSave) 52 | 53 | db.event.on('github_error', (err, res) => { 54 | console.log('github error') 55 | console.log(err) 56 | console.log() 57 | }) 58 | 59 | db.event.on('file_error', (err, file) => { 60 | console.log('file error') 61 | console.log('FILE: ' + file.id) 62 | console.log('ERROR: ' + err.code) 63 | console.log() 64 | }) 65 | 66 | db.event.on('refreshing', () => { 67 | // MIGHT WANT TO LOCK DOWN THINGS FOR A BIT 68 | console.log('LETS DO THIS') 69 | }) 70 | 71 | db.event.on('refreshed', (err) => { 72 | if (err) { 73 | // error should be handled 74 | console.log(err) 75 | } 76 | console.log('refresh done') 77 | db().each((file) => { 78 | console.log(file.id) 79 | if (!file.error) { 80 | console.log('RAW: ' + file.raw.length + ' | HTML: ' + file.html.length) 81 | console.log() 82 | } else { 83 | console.log('ERROR') 84 | console.log() 85 | } 86 | }) 87 | }) 88 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # :cloud: Gist-DB :cloud: 2 | 3 | Treat your gist account like a database. Powered by [TaffyDB][taffydb-link] 4 | and [Github][node-github-link]. 5 | 6 | [![npm downloads](https://img.shields.io/npm/dm/gist-db.svg)](https://www.npmjs.com/package/gist-db) 7 | [![npm version](https://img.shields.io/npm/v/gist-db.svg)](https://www.npmjs.com/package/gist-db) 8 | [![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg)](http://standardjs.com/) 9 | 10 | ## Install 11 | 12 | ```sh 13 | npm install --save gist-db 14 | ``` 15 | 16 | ## Usage 17 | 18 | ```js 19 | var config = { 20 | github: { 21 | username: "mcwhittemore" 22 | } 23 | } 24 | 25 | var db = require("gist-db")(config); 26 | 27 | db.event.on('refreshing', function(){ 28 | //MIGHT WANT TO LOCK DOWN THINGS FOR A BIT 29 | console.log("LETS DO THIS"); 30 | }); 31 | 32 | db.event.on('refreshed', function(err){ 33 | db().each(function(file){ 34 | console.log(file); 35 | }); 36 | }); 37 | ``` 38 | 39 | ## File Object structure 40 | 41 | ```js 42 | { 43 | id = gist_id + "_" + filename, 44 | filename = filename, 45 | gist_id = gist_id, 46 | error: undefined, 47 | raw: "THE RAW VALUE OF THE FILE", 48 | type: "mime type", 49 | language: "language the file is written in", 50 | raw_url: "https path to the raw text version of the file", 51 | size: numeric size of the file, 52 | gist: { 53 | id: gist_id 54 | public: boolean, 55 | created_at: date object, 56 | updated_at: date object, 57 | description: "the gist description" 58 | } 59 | } 60 | ``` 61 | 62 | ## Demo 63 | 64 | ```sh 65 | # Clone the repository 66 | git clone https://github.com/pinn3/gist-db 67 | 68 | # Run the demo! This will also install dependencies and run the build script 69 | npm run demo 70 | ``` 71 | 72 | ## API 73 | 74 | Please refer to the [TaffyDB docs](http://www.taffydb.com/workingwithdata) for 75 | more details 76 | 77 | ### GISTDB(config, fileInit, fileSave) 78 | 79 | Create a new gist-db. 80 | 81 | **Parameters** 82 | 83 | * config: A settings object. 84 | 85 | ```js 86 | Required: { 87 | github:{ 88 | username:"SOME_USER_NAME" 89 | } 90 | } 91 | 92 | Defaults: { 93 | refreshMin: 10, 94 | github: { 95 | per_page: 100, 96 | timeout: 5000, 97 | version: "3.0.0" 98 | }, 99 | local: { 100 | save: "NEVER", //NEVER, ON_REFRESH, ALWAYS 101 | location: undefined 102 | } 103 | } 104 | 105 | Available: { 106 | refreshMin: 10, 107 | github: { 108 | per_page: 100, 109 | timeout: 5000 110 | version: "3.0.0", 111 | authenticate: { 112 | type: "basic or oauth", 113 | username: "your_username_if_basic", 114 | password: "your_password_if_basic", 115 | token: "your_oauth_token_if_oauth" 116 | }, 117 | local: { 118 | save: "NEVER OR NEVER OR ON_REFRESH OR ALWAYS" 119 | location: "path and filename" 120 | } 121 | } 122 | } 123 | ``` 124 | 125 | * fileInit: function(file). A function that returns the file obj if it should 126 | be added to the DB and undefined if it should be excluded. 127 | 128 | * fileSave: function(file, callback). A function that allows for further 129 | parameter work on files after the raw data has been received. A functioning 130 | implementation of this MUST pass the file object as a parameter to callback 131 | to save changes to the DB. Note: this will perform an update to the 132 | database. 133 | 134 | ### db({field:value}) 135 | 136 | * TaffyDB: Yes 137 | 138 | * Returns: All rows that meet the passed criteria. Not passing an object, 139 | will return all rows. 140 | 141 | ### db.insert({}) 142 | 143 | Inserts records into the database. 144 | 145 | * TaffyDB: Yes 146 | * Returns: A query pointing to the inserted records 147 | 148 | ### db.github 149 | 150 | Full use of the github module passed the github subsection of your config file. 151 | 152 | ### db.event 153 | 154 | An implementation of require("events").EventEmitter 155 | 156 | ### db.event.on('refreshing', function(){}) 157 | 158 | Use to be notified when gist-db is connecting gist for a refresh. 159 | 160 | ### db.event.on('refreshed', function(err){}) 161 | 162 | Use to be notified when gist-db is done its current refresh. If err is set, 163 | this refresh was ended due to error. 164 | 165 | ### db.event.on('file_error', function(err, file){}) 166 | 167 | Use to be notified of errors in gathering data on the gist files. 168 | 169 | **Parameters** 170 | 171 | * err: The error object that triggered this event 172 | * file: The file object that was being gathered when the error occurred 173 | 174 | ### db.event.on('github_error', function(err, res){}) 175 | 176 | Use to be notified of errors when connecting with github. 177 | 178 | **Parameters** 179 | 180 | * err: the github module error object that triggered this event 181 | 182 | * res: The github module response object. Might contain good data about the 183 | error. 184 | 185 | ## Things to be done 186 | 187 | ### 0.1.10 188 | 189 | * Add tests 190 | 191 | ### 0.2.0 192 | 193 | * Add Update gist.github 194 | * Add Insert gist.github 195 | * Add Delete gist.github 196 | 197 | ## Licenses 198 | 199 | All code not otherwise specified is released under the MIT License. 200 | 201 | All code found in the node_modules directory is Copyrighted by its creators. 202 | Please see each module for further details. 203 | 204 | [taffydb-link]: http://www.taffydb.com/ 205 | [node-github-link]: https://github.com/mikedeboer/node-github 206 | -------------------------------------------------------------------------------- /src/gist-db.js: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import url from 'url' 3 | import GitHubApi from 'github' 4 | import { taffy } from 'taffydb' 5 | import { EventEmitter } from 'events' 6 | import request from 'request' 7 | import config from './config' 8 | 9 | let finalConfig = config 10 | let db = null 11 | let githubMeta = null 12 | let fileInit = null 13 | let fileSave = null 14 | let lastCall = null 15 | let numGistPending = -1 16 | 17 | export default (userConfig, userFileInit, userFileSave) => { 18 | // merge configs 19 | if (typeof userConfig === 'object') { 20 | finalConfig = mergeConfigs(config, userConfig) 21 | } else if (typeof userConfig === 'function') { 22 | userFileInit = userConfig 23 | userConfig = {} 24 | } 25 | 26 | if (typeof userFileInit === 'object' || !userFileInit) { 27 | userFileInit = (file) => { return file } 28 | } 29 | 30 | if (typeof userFileSave === 'object' || !userFileSave) { 31 | userFileSave = (file, callback) => {} 32 | } 33 | 34 | fileInit = userFileInit 35 | fileSave = userFileSave 36 | 37 | db = initDb() 38 | 39 | db.event = new EventEmitter() 40 | 41 | // ADD REFRESH FUNCTION TO db 42 | db.refresh = refresh 43 | 44 | // CONNECT TO GITHUB 45 | db.github = new GitHubApi({ 46 | version: finalConfig.github.version, 47 | timeout: config.github.timeout 48 | }) 49 | 50 | if (finalConfig.github.authenticate) { 51 | db.github.authenticate(finalConfig.github.authenticate) 52 | } 53 | 54 | // CREATE EVENTS 55 | 56 | // START TIMER 57 | runRefresh() 58 | 59 | return db 60 | } 61 | 62 | const initDb = () => { 63 | if (finalConfig.local.save !== 'NEVER') { 64 | if (!finalConfig.local.save) { 65 | throw new Error('config.local.save was not set') 66 | } 67 | 68 | if (!finalConfig.local.location) { 69 | throw new Error('config.local.location was not set') 70 | } 71 | 72 | try { 73 | const localDb = JSON.parse(fs.readFileSync(finalConfig.local.location)) 74 | if (localDb && !Array.isArray(localDb)) { 75 | throw new Error('Local database was not an array') 76 | } 77 | 78 | return taffy(localDb) 79 | } catch (err) { 80 | if (err.code !== 'ENOENT') { 81 | throw err 82 | } 83 | } 84 | } 85 | 86 | return taffy([]) 87 | } 88 | 89 | const saveDb = () => { 90 | // GATHER DATA 91 | const gistStorage = db().get() 92 | 93 | // SAVE DATA 94 | fs.writeFile(finalConfig.local.location, JSON.stringify(gistStorage)) 95 | } 96 | 97 | const mergeConfigs = (keep, add) => { 98 | const keys = Object.keys(add) 99 | 100 | for (const key of keys) { 101 | if (!keep[key] || typeof add[key] !== 'object') { 102 | keep[key] = add[key] 103 | } else { 104 | keep[key] = mergeConfigs(keep[key], add[key]) 105 | } 106 | } 107 | 108 | return keep 109 | } 110 | 111 | const runRefresh = () => { 112 | db.event.emit('refreshing') 113 | refresh(1) 114 | setTimeout(runRefresh, finalConfig.refreshMin * 1000 * 60) 115 | } 116 | 117 | const refresh = (pageNum) => { 118 | trackPendingGists(true, 'start of refresh') 119 | 120 | if (!pageNum) { 121 | pageNum = 1 122 | } 123 | 124 | const options = { 125 | user: finalConfig.github.username, 126 | per_page: finalConfig.github.per_page, 127 | page: pageNum 128 | } 129 | 130 | if (lastCall) { 131 | options.since = lastCall 132 | } 133 | 134 | db.github.gists.getFromUser(options, callGithub) 135 | 136 | trackPendingGists(false, 'end of refresh') 137 | } 138 | 139 | const continueRefresh = () => { 140 | if (githubMeta && githubMeta.link) { 141 | const links = githubMeta.link.split(', ') 142 | 143 | let next = -1 144 | 145 | for (const linkTag of links) { 146 | const linkParts = linkTag.split('; ') 147 | 148 | let link = linkParts[0] 149 | link = link.substring(1, link.length - 1) 150 | const details = url.parse(link, true) 151 | 152 | if (linkParts[1] === 'rel="next"') { 153 | next = details.query.page 154 | } 155 | } 156 | 157 | // FIGURE OUT HOW TO DO THIS IN A LOOP 158 | // SO A BUNCH CAN GO AT ONCE 159 | if (next > -1) { 160 | console.log('NEXT PAGE: ' + next) 161 | refresh(next) 162 | } 163 | } 164 | } 165 | 166 | const endRefresh = (err) => { 167 | db.event.emit('refreshed', err) 168 | lastCall = (new Date()).toISOString() 169 | if (finalConfig.local.save !== 'NEVER') { 170 | saveDb(db) 171 | } 172 | } 173 | 174 | const callGithub = (err, res) => { 175 | trackPendingGists(true, 'start of callGithub') 176 | 177 | if (err) { 178 | db.event.emit('github_error', err, res) 179 | endRefresh(err) 180 | } else { 181 | githubMeta = res.meta 182 | delete res.meta 183 | 184 | continueRefresh() 185 | 186 | trackPendingGists(true, 'gather Github Info callGithub') 187 | gatherGithubInfo(res, 0, 0) 188 | } 189 | 190 | trackPendingGists(false, 'end of callGithub') 191 | 192 | if (numGistPending === 0) { 193 | endRefresh() 194 | } 195 | } 196 | 197 | const getRawFile = (err, res, body, fileParams) => { 198 | // GATHER RAW AND SAVE FILE TO DB 199 | if (err) { 200 | fileParams.file.error = 'dropped_raw_file' 201 | fileParams.file.raw = null 202 | 203 | db.event.emit('file_error', err, fileParams.file) 204 | } else { 205 | fileParams.file.error = null 206 | fileParams.file.raw = body 207 | } 208 | 209 | db.merge(fileParams.file) 210 | fileSave(fileParams.file, (theFile) => { 211 | db.merge(theFile) 212 | }) 213 | 214 | trackPendingGists(false, 'got file getRawFile') 215 | 216 | if (fileParams.fileIndex === fileParams.filenames.length - 1 && fileParams.gistIndex === fileParams.gists.length - 1 && numGistPending === 0) { 217 | endRefresh() 218 | } else { 219 | trackPendingGists(true, 'get raw file getRawFile') 220 | gatherGithubInfo(fileParams.gists, fileParams.gistIndex, fileParams.fileIndex + 1) 221 | } 222 | } 223 | 224 | const gatherGithubInfo = (gists, gistIndex, fileIndex) => { 225 | if (gistIndex < gists.length) { 226 | const gist = gists[gistIndex] 227 | 228 | const filenames = Object.keys(gist.files) 229 | 230 | if (fileIndex < filenames.length) { 231 | const filename = filenames[fileIndex] 232 | const rawFileUrl = gist.files[filename].raw_url 233 | 234 | let file = gist.files[filename] 235 | file.id = gist.id + '_' + file.filename 236 | file.gist_id = gist.id 237 | file.gist = { 238 | id: gist.id, 239 | public: gist.public, 240 | created_at: new Date(gist.created_at), 241 | updated_at: new Date(gist.updated_at), 242 | description: gist.description 243 | } 244 | 245 | file = fileInit(file) // returns null if it shouldn't be in the DB 246 | 247 | let oldFile = null 248 | if (file) { 249 | oldFile = db({id: file.id}).first() 250 | if (!oldFile) { 251 | oldFile = null 252 | } 253 | } 254 | 255 | if (file && (!oldFile || file.gist.updated_at.getTime() > new Date(oldFile.gist.updated_at).getTime())) { 256 | const fileParams = { 257 | file: file, 258 | fileIndex: fileIndex, 259 | filenames: filenames, 260 | gists: gists, 261 | gistIndex: gistIndex 262 | } 263 | trackPendingGists(true, 'get raw file gatherGithubInfo') 264 | request({uri: rawFileUrl}, (err, res, body) => { getRawFile(err, res, body, fileParams) }) 265 | } else { 266 | trackPendingGists(true, 'next file gatherGithubInfo') 267 | gatherGithubInfo(gists, gistIndex, fileIndex + 1) 268 | } 269 | } else { 270 | trackPendingGists(true, 'next gist gatherGithubInfo') 271 | gatherGithubInfo(gists, gistIndex + 1, 0) 272 | } 273 | } 274 | 275 | trackPendingGists(false, 'end of gatherGithubInfo') 276 | if (numGistPending === 0) { 277 | endRefresh() 278 | } 279 | } 280 | 281 | const trackPendingGists = (add, note) => { 282 | if (add) { 283 | numGistPending === -1 ? numGistPending = 1 : numGistPending++ 284 | } else { 285 | numGistPending-- 286 | } 287 | // console.log("ADD: "+add+" | NOTE: "+note + " | NUM: "+numGistPending) 288 | } 289 | --------------------------------------------------------------------------------