├── .gitignore ├── README.md ├── cli ├── helpers │ ├── autocomplete-facts.js │ ├── autocomplete-rules.js │ ├── cached.js │ ├── edit.js │ └── format.js ├── package.json ├── pf-checkpoint.js ├── pf-compute.js ├── pf-facts.js ├── pf-rules.js └── pf.js ├── core ├── checkpoints.js ├── compute.js ├── db.js ├── facts.js ├── helpers │ ├── date.js │ └── unicode-letter.js ├── package.json ├── parser-parser-test.js ├── parser-parser.js ├── parser.js └── rules.js ├── lerna.json ├── package.json └── screencast.gif /.gitignore: -------------------------------------------------------------------------------- 1 | *.swo 2 | *.swp 3 | node_modules 4 | *.env 5 | package-lock.json 6 | _database 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pf 2 | 3 | ![](screencast.gif) 4 | 5 | **pf** is a (for now) a command line tool that let's you turn sentences into structured data of any format you want by using a simple parser syntax. 6 | 7 | Let's say you have a fabric store, a physical fabric store, and you want to keep track of all your finances and inventory. The only information you have are the raw facts that happened at your store, as they happen, and that's the only thing you need. So imagine that during the day you'll add facts like the following: 8 | 9 | ``` 10 | pf facts add 'sold 2m of cashmere for 104 bucks' 11 | pf facts add 'sold 4m of wool for 150 bucks' 12 | pf facts add 'got 100m of cashmere delivered' 13 | pf facts add 'paid 200 to cashmere supplier' 14 | pf facts add 'sold 6m of cashmere for 312 bucks' 15 | pf facts add 'paid 250 to employee Maria' 16 | ``` 17 | 18 | After that (or before, it doesn't matter) you would define rules (by calling `pf rules add` or `pf rules edit`) to handle the 3 kinds of facts listed above with patterns somewhat like 19 | 20 | ``` 21 | 'sold m of for bucks' 22 | 'got m of delivered' 23 | 'paid to ' 24 | ``` 25 | 26 | When you call `pf compute` each fact is run, in order, through each available pattern and, if matched, the rule script is run. Each rule script receive the parameters captured in the fact parsing and the fact timestamp and can read and modify the current `state` (which begins as an empty JSON object `{}` by default). So for the "sold ..." rule you would perhaps 27 | 28 | * add the sale data to a big list of sales 29 | * add the value to your cash balances 30 | * subtract the sold items from your inventory 31 | 32 | for the "got ..." rule you would 33 | 34 | * add the received items to the inventory 35 | 36 | and for the "paid ..." rule you would 37 | 38 | * subtract the paid value from the cash balances 39 | * if the value was paid to an employee, keep track of your payments to that employee 40 | 41 | Then you end up with a JSON object with all the data you need and you can use different tools, like [jq](https://stedolan.github.io/jq/), to crunch the data and produce reports, charts, prints and other useful stuff. 42 | 43 | ```json 44 | { 45 | "days": { 46 | "2018-02-30": { 47 | "sales": [ 48 | { 49 | "item": "cashmere", 50 | "q": 2, 51 | "amount": 104 52 | }, 53 | { 54 | "item": "wool", 55 | "q": 4, 56 | "amount": 150 57 | }, 58 | { 59 | "item": "cashmere", 60 | "q": 6, 61 | "amount": 312 62 | } 63 | ], 64 | "payments": [ 65 | { 66 | "target": "cashmere supplier", 67 | "amount": 200 68 | }, 69 | { 70 | "target": "maria", 71 | "amount": 250 72 | } 73 | ], 74 | "balance": 116 75 | } 76 | }, 77 | "inventory": { 78 | "cashmere": 177, 79 | "wool": 328 80 | }, 81 | "employees": { 82 | "maria": { 83 | "monthly_payments": { 84 | "2018-02": [ 85 | 200, 86 | 250 87 | ] 88 | }, 89 | "outstanding_balance": 550 90 | } 91 | } 92 | } 93 | ``` 94 | 95 | If you decide later to start keeping track of things in a different way or to change your schema entirely, you just have to rewrite your rules, the facts will continue to be the same. 96 | 97 | 98 | ### install 99 | 100 | ``` 101 | npm install -g pf-cli 102 | ``` 103 | 104 | --- 105 | 106 | This is an idea under development. The CLI program works, but much more functionality will be added yet, most notoriously: 107 | 108 | * [ ] incremental computing of facts 109 | * [ ] initial data loading and checkpoint saving 110 | * [ ] better logs and rule debugging helpers 111 | * [ ] [PouchDB](http://pouchdb.com/) sync 112 | * [ ] a web client, at least for fact inputting, so normal people can use it on predefined schemas 113 | * [ ] fact input autosuggest based on the rule patterns structure 114 | -------------------------------------------------------------------------------- /cli/helpers/autocomplete-facts.js: -------------------------------------------------------------------------------- 1 | const Sifter = require('sifter') 2 | const Autocomplete = require('prompt-autocompletion') 3 | 4 | const {listFacts} = require('pf-core/facts') 5 | 6 | const {formatLine} = require('./format') 7 | 8 | module.exports = async function factsAutocompleter (message) { 9 | let facts = await listFacts() 10 | if (!facts) { 11 | console.log('Your database has 0 facts.') 12 | return 13 | } 14 | 15 | let sifter = new Sifter(facts) 16 | let autocomplete = new Autocomplete({ 17 | type: 'autocomplete', 18 | name: 'fact', 19 | message: message, 20 | source: async (_, input) => { 21 | input = input || '' 22 | let r = sifter.search(input, {fields: ['line'], limit: 23, sort_empty: '_id desc'}) 23 | return r.items 24 | .map(item => facts[item.id]) 25 | .map(doc => ({value: doc._id, name: formatLine(doc)})) 26 | } 27 | }) 28 | 29 | return autocomplete.run() 30 | } 31 | -------------------------------------------------------------------------------- /cli/helpers/autocomplete-rules.js: -------------------------------------------------------------------------------- 1 | const Sifter = require('sifter') 2 | const Autocomplete = require('prompt-autocompletion') 3 | 4 | const {listRules} = require('pf-core/rules') 5 | 6 | const {formatRule} = require('./format') 7 | 8 | module.exports = async function rulesAutocompleter (message) { 9 | let rules = await listRules() 10 | if (!rules) { 11 | console.log('Your database has 0 rules.') 12 | return 13 | } 14 | 15 | let sifter = new Sifter(rules) 16 | let autocomplete = new Autocomplete({ 17 | type: 'autocomplete', 18 | name: 'rule', 19 | message: message, 20 | source: async (_, input) => { 21 | input = input || '' 22 | let r = sifter.search(input, {fields: ['line'], limit: 23, sort_empty: '_id desc'}) 23 | return r.items 24 | .map(item => rules[item.id]) 25 | .map(doc => ({value: doc._id, name: formatRule(doc)})) 26 | } 27 | }) 28 | 29 | return autocomplete.run() 30 | } 31 | -------------------------------------------------------------------------------- /cli/helpers/cached.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | const path = require('path') 3 | const mkdirp = require('mkdirp') 4 | 5 | const _path = path.join('/tmp', process.cwd()) 6 | const _file = path.join(_path, 'cached.json') 7 | 8 | module.exports.path = _path 9 | module.exports.file = _file 10 | 11 | module.exports.read = function () { 12 | try { 13 | let d = fs.readFileSync(_file, 'utf-8') 14 | return JSON.parse(d) 15 | } catch (e) { 16 | return {state: {}, lastFact: undefined} 17 | } 18 | } 19 | 20 | module.exports.reset = function () { 21 | mkdirp.sync(_path) 22 | return fs.writeFileSync(_file, '{}', {encoding: 'utf-8'}) 23 | } 24 | 25 | module.exports.write = function (state, lastFact) { 26 | mkdirp.sync(_path) 27 | return fs.writeFileSync(_file, JSON.stringify({ 28 | state, 29 | lastFact 30 | }), {encoding: 'utf-8'}) 31 | } 32 | -------------------------------------------------------------------------------- /cli/helpers/edit.js: -------------------------------------------------------------------------------- 1 | const editor = require('editor') 2 | const tempfile = require('tempfile') 3 | const fs = require('fs') 4 | 5 | module.exports = async function edit (contents, ext = '.txt') { 6 | let filepath = tempfile(ext) 7 | fs.writeFileSync(filepath, contents, {encoding: 'utf8'}) 8 | 9 | return new Promise((resolve, reject) => editor(filepath, (code, sig) => { 10 | if (code !== 0) return reject(sig) 11 | 12 | fs.readFile(filepath, {encoding: 'utf8'}, (err, editedContents) => { 13 | if (err) return reject(err) 14 | resolve(editedContents) 15 | }) 16 | })) 17 | } 18 | -------------------------------------------------------------------------------- /cli/helpers/format.js: -------------------------------------------------------------------------------- 1 | const {bold, cyan} = require('chalk') 2 | 3 | const {formatId} = require('pf-core/helpers/date') 4 | 5 | module.exports.formatLine = fact => { 6 | let lines = fact.line.split('\n') 7 | let first = lines[0] 8 | let extra = lines.length - 1 9 | return `${cyan(formatId(fact._id))} ${bold('::')} ${first}${extra ? ` (+${extra} lines)` : ''}` 10 | } 11 | 12 | module.exports.formatRule = rule => `${cyan(rule._id)} ${bold('::')} ${rule.pattern}` 13 | 14 | module.exports.formatCheckpoint = chk => { 15 | return `${cyan(formatId(chk._id))} ${bold('::')} JSON with ${JSON.stringify(chk.state).length} characters.` 16 | } 17 | -------------------------------------------------------------------------------- /cli/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pf-cli", 3 | "version": "0.1.1", 4 | "description": "", 5 | "main": "pf.js", 6 | "license": "ISC", 7 | "bin": { 8 | "pf": "pf.js" 9 | }, 10 | "dependencies": { 11 | "chalk": "^2.1.0", 12 | "commander": "^2.11.0", 13 | "debug": "^3.0.0", 14 | "editor": "^1.0.0", 15 | "jsondiffpatch": "^0.2.4", 16 | "mkdirp": "^0.5.1", 17 | "pf-core": "^0.1.1", 18 | "prompt-autocompletion": "^0.1.1", 19 | "sifter": "^0.5.2", 20 | "supports-color": "^4.2.1", 21 | "tempfile": "^2.0.0" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /cli/pf-checkpoint.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | var program = require('commander') 3 | 4 | const {addCheckpoint, listCheckpoints, fetchCheckpoint, 5 | delCheckpoint, updateCheckpoint} = require('pf-core/checkpoints') 6 | 7 | const {formatCheckpoint} = require('./helpers/format') 8 | 9 | program 10 | .command('list') 11 | .description('list checkpoints') 12 | .option('-n ', 'last checkpoints, defaults to 23.', parseInt) 13 | .action(async cmd => { 14 | let n = cmd.n || 23 15 | let facts = await listCheckpoints() 16 | facts 17 | .slice(-n) 18 | .forEach(fact => console.log(formatCheckpoint(fact))) 19 | }) 20 | 21 | program 22 | .parse(process.argv) 23 | -------------------------------------------------------------------------------- /cli/pf-compute.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | const program = require('commander') 3 | const debug = require('debug')('pf-cli:pf-compute') 4 | 5 | const compute = require('pf-core/compute') 6 | const cached = require('./helpers/cached') 7 | 8 | program 9 | .option('-r, --reset', 'compute from scratch') 10 | .description(`by default, this will look for a temporary file where the supposedly last updated version of the computed state has been stored and compute from that. If it is not found, then it will compute from scratch. You can force this behavior by passing -r.`) 11 | .parse(process.argv) 12 | 13 | async function main (cmd) { 14 | var initial = {} 15 | var lastFact 16 | 17 | if (cmd.reset) { 18 | debug('will reset.') 19 | } else { 20 | debug('will read from cache.') 21 | 22 | let current = cached.read() 23 | initial = current.state 24 | lastFact = current.lastFact 25 | 26 | debug(`state read: %j`, initial) 27 | debug(`last fact read: ${lastFact}`) 28 | } 29 | 30 | if (!initial) { 31 | initial = {} 32 | lastFact = undefined 33 | } 34 | 35 | let {state, last} = await compute.from(initial, lastFact) 36 | 37 | console.log(JSON.stringify(state)) 38 | cached.write(state, last) 39 | } 40 | 41 | main(program) 42 | -------------------------------------------------------------------------------- /cli/pf-facts.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | const program = require('commander') 3 | const formatterConsole = require('jsondiffpatch/src/formatters/console') 4 | const {yellow, bgYellow, gray, red, green, blue} = require('chalk') 5 | 6 | const {addFact, listFacts, fetchFact, delFact, updateFact} = require('pf-core/facts') 7 | const compute = require('pf-core/compute') 8 | 9 | const {formatLine, formatRule} = require('./helpers/format') 10 | const autocompleteFacts = require('./helpers/autocomplete-facts') 11 | const editFile = require('./helpers/edit') 12 | const cached = require('./helpers/cached') 13 | 14 | program 15 | .command('list') 16 | .description('list facts') 17 | .option('-n ', 'last facts, defaults to 23.', parseInt) 18 | .action(async cmd => { 19 | let n = cmd.n || 23 20 | let facts = await listFacts() 21 | facts 22 | .slice(-n) 23 | .forEach(fact => console.log(formatLine(fact))) 24 | }) 25 | 26 | program 27 | .command('add ') 28 | .description('add a new fact') 29 | .option('-d, --date ', 30 | 'add this fact with the given date, instead of now.', d => new Date(d)) 31 | .action(async (line, cmd) => { 32 | // compute everything till this point so we can compute this fact 33 | let prev = cached.read() 34 | let res = await compute.from(prev.state, prev.lastFact) 35 | 36 | // now add this fact 37 | let fact = await addFact(line, cmd.date) 38 | console.log(` ${yellow('#')} added '${bgYellow.black(line)}'.`) 39 | 40 | // then compute this fact 41 | let reducers = await compute.reducers() 42 | let {state, matched, errors, diff} = await compute.next(res.state, reducers, fact) 43 | 44 | // write to cache 45 | cached.write(state, fact._id) 46 | 47 | // print all the info 48 | matched.forEach(rule => 49 | console.log(` ${green('>')} matched rule ${formatRule(rule)}`) 50 | ) 51 | 52 | errors.forEach(err => 53 | console.log(` ${red('>')} error: ${err}`) 54 | ) 55 | 56 | let f = formatterConsole.format(diff).split('\n').map(l => ' ' + l).join('\n') 57 | console.log(` ${blue('>')} diff: ${gray(f)}`) 58 | }) 59 | 60 | program 61 | .command('edit [fact_id]') 62 | .description('edit a fact') 63 | .action(async factId => { 64 | factId = factId || await autocompleteFacts('Select a fact to edit:') 65 | 66 | let fact = await fetchFact(factId) 67 | let newcontents = await editFile(`${fact.line} 68 | 69 | # replace the line(s) above with the new, updated version of the fact. 70 | # a fact can span through multiple lines. 71 | # lines starting with a hash will be ignored. 72 | # if there are no valid lines, this update will be ignored. 73 | `) 74 | 75 | let newline = newcontents 76 | .split('\n') 77 | .map(line => line.trim()) 78 | .filter(line => line[0] && line[0] !== '#') 79 | .join('\n') 80 | 81 | if (!newline || newline === fact.line) { 82 | console.log('ignoring update.') 83 | return 84 | } 85 | 86 | fact.line = newline 87 | updateFact(fact) 88 | .then(cached.reset) 89 | .then(() => console.log(`updated ${factId}.`)) 90 | .catch(e => console.error(e)) 91 | }) 92 | 93 | program 94 | .command('del [fact_id]') 95 | .description('remove a fact') 96 | .action(async () => { 97 | var factId 98 | if (program.args.length === 0) { 99 | factId = await autocompleteFacts('Select a fact to delete:') 100 | } else { 101 | factId = program.args[0] 102 | } 103 | 104 | fetchFact(factId).then(delFact) 105 | .then(cached.reset) 106 | .then(() => console.log(`removed '${factId}'.`)) 107 | .catch(e => console.error(e)) 108 | }) 109 | 110 | program 111 | .parse(process.argv) 112 | -------------------------------------------------------------------------------- /cli/pf-rules.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | const program = require('commander') 3 | 4 | const {addRule, listRules, fetchRule, delRule, updateRule} = require('pf-core/rules') 5 | const autocompleteRules = require('./helpers/autocomplete-rules') 6 | const editFile = require('./helpers/edit') 7 | const {formatRule} = require('./helpers/format') 8 | const cached = require('./helpers/cached') 9 | 10 | program 11 | .command('list') 12 | .description('list rules') 13 | .option('-n ', 'last rules, defaults to 23.', parseInt) 14 | .action(async cmd => { 15 | let n = cmd.n || 23 16 | let rules = await listRules() 17 | rules 18 | .slice(-n) 19 | .forEach(rule => console.log(formatRule(rule))) 20 | }) 21 | 22 | program 23 | .command('add [kind]') 24 | .description('create a new rule') 25 | .option('--kind ', "language in which the rule will be written. defaults to 'js'") 26 | .action(async (_, cmd) => { 27 | let kind = cmd.kind || 'js' 28 | var pattern 29 | 30 | if (kind === 'js') { 31 | let newcontents = await editFile(`/* 32 | * pattern: has paid 33 | */ 34 | 35 | module.exports = function (state, params, timestamp) { 36 | state.paid = state.paid || {} 37 | state.paid[params.name] = state.paid[name] || [] 38 | state.paid[params.name].push({when: timestamp, amount: currency}) 39 | }`, '.js') 40 | if (newcontents.trim() === '') { 41 | console.log('aborting.') 42 | return 43 | } 44 | 45 | try { 46 | pattern = newcontents.match(/pattern\s*:(.+)/)[1].trim() 47 | } catch (e) { 48 | console.log(newcontents) 49 | console.error('missing pattern.') 50 | return 51 | } 52 | 53 | console.log(kind, pattern, newcontents) 54 | addRule(kind, pattern, newcontents) 55 | .then(cached.reset) 56 | .then(() => console.log('rule added.')) 57 | .catch(e => console.error(e)) 58 | return 59 | } 60 | 61 | console.log(`kind ${kind} not supported yet.`) 62 | }) 63 | 64 | program 65 | .command('show [rule_id]') 66 | .description('see a rule') 67 | .action(async ruleId => { 68 | ruleId = ruleId || await autocompleteRules('Select a rule to edit:') 69 | 70 | let rule = await fetchRule(ruleId) 71 | console.log(`id: ${rule._id} 72 | rev: ${rule._rev} 73 | pattern: ${rule.pattern} 74 | kind: ${rule.kind} 75 | 76 | ${rule.code} 77 | `) 78 | }) 79 | 80 | program 81 | .command('edit [rule_id]') 82 | .description('edit a rule') 83 | .action(async ruleId => { 84 | ruleId = ruleId || await autocompleteRules('Select a rule to edit:') 85 | 86 | let rule = await fetchRule(ruleId) 87 | let newcontents = await editFile(rule.code, '.js') 88 | var pattern 89 | 90 | try { 91 | pattern = newcontents.match(/pattern\s*:(.+)/)[1].trim() 92 | } catch (e) { 93 | console.log(newcontents) 94 | console.error('missing pattern. ignoring update.') 95 | return 96 | } 97 | 98 | if (rule.pattern === pattern && rule.code === newcontents) { 99 | console.log('nothing has changed.') 100 | return 101 | } 102 | 103 | rule.pattern = pattern 104 | rule.code = newcontents 105 | 106 | updateRule(rule) 107 | .then(cached.reset) 108 | .then(() => console.log(`updated ${ruleId}.`)) 109 | .catch(e => console.error(e)) 110 | }) 111 | 112 | program 113 | .command('del [rule_id]') 114 | .description('remove a rule') 115 | .action(async () => { 116 | var ruleId 117 | if (program.args.length === 0) { 118 | ruleId = await autocompleteRules('Select a rule to delete:') 119 | } else { 120 | ruleId = program.args[0] 121 | } 122 | 123 | fetchRule(ruleId).then(delRule) 124 | .then(cached.reset) 125 | .then(() => console.log(`removed '${ruleId}'.`)) 126 | .catch(e => console.error(e)) 127 | }) 128 | 129 | program 130 | .parse(process.argv) 131 | -------------------------------------------------------------------------------- /cli/pf.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | const program = require('commander') 3 | 4 | program 5 | .version('0.1.0') 6 | .command('facts', 'create, view, update and delete facts') 7 | .command('rules', 'create, view, update and delete rules') 8 | .command('compute', 'compute everything from the last checkpoint') 9 | .command('checkpoint', 'save a checkpoint with the current state') 10 | .parse(process.argv) 11 | -------------------------------------------------------------------------------- /core/checkpoints.js: -------------------------------------------------------------------------------- 1 | const debug = require('debug')('pf-core:checkpoints') 2 | 3 | const db = require('./db') 4 | const compute = require('./compute') 5 | 6 | module.exports.listCheckpoints = listCheckpoints 7 | async function listCheckpoints () { 8 | let res = db.allDocs({ 9 | startkey: 'chk:', 10 | endkey: 'chk:~', 11 | include_docs: true 12 | }) 13 | return res.rows.map(r => r.doc) 14 | } 15 | 16 | module.exports.lastCheckpoint = lastCheckpoint 17 | async function lastCheckpoint () { 18 | let res = db.allDocs({ 19 | descending: true, 20 | startkey: 'chk:~', 21 | endkey: 'chk:', 22 | limit: 1, 23 | include_docs: true 24 | }) 25 | 26 | if (res.rows.length) { 27 | return res.rows[0].doc 28 | } 29 | } 30 | 31 | module.exports.addCheckpoint = addCheckpoint 32 | async function addCheckpoint (content, time) { 33 | var doc = {} 34 | 35 | if (content && time) { 36 | debug('content and time are given, save a checkpoint with them.') 37 | 38 | if (isNaN(time.getDate())) { 39 | throw new Error('date is invalid.') 40 | } 41 | 42 | doc.state = content 43 | doc._id = `chk:${parseInt(time.getTime() / 1000)}` 44 | } else { 45 | debug('content and time are not given, use the current state.') 46 | 47 | let {state, last} = compute.from({}) 48 | doc.state = state 49 | doc._id = 'chk:' + last.split(':')[1] 50 | } 51 | 52 | let res = db.put(doc) 53 | doc._rev = res.rev 54 | return doc 55 | } 56 | 57 | module.exports.fetchCheckpoint = fetchCheckpoint 58 | async function fetchCheckpoint (id) { return db.get(id) } 59 | 60 | module.exports.updateCheckpoint = updateCheckpoint 61 | async function updateCheckpoint (fact) { return db.put(fact) } 62 | 63 | module.exports.delCheckpoint = delCheckpoint 64 | async function delCheckpoint (fact) { return db.remove(fact) } 65 | -------------------------------------------------------------------------------- /core/compute.js: -------------------------------------------------------------------------------- 1 | const debug = require('debug')('pf-core:compute') 2 | const jsondiffpatch = require('jsondiffpatch') 3 | 4 | const {listRules} = require('./rules') 5 | const {listFacts} = require('./facts') 6 | const patternParser = require('./parser-parser') 7 | const {makeLineParser} = require('./parser') 8 | 9 | const compute = { 10 | from, 11 | next, 12 | reducers 13 | } 14 | 15 | async function from (state, lastFact = 'f:0') { 16 | let nextTimestamp = parseInt(lastFact.split(':')[1]) + 1 17 | 18 | let facts = await listFacts('f:' + nextTimestamp) 19 | 20 | let reducers = await compute.reducers() 21 | debug(`starting computation with state ${JSON.stringify(state)}, ${reducers.length} reducers and ${facts.length} facts.`) 22 | 23 | // process all the facts 24 | var fact 25 | for (let i = 0; i < facts.length; i++) { 26 | fact = facts[i] 27 | debug(`computing fact ${fact.line}`) 28 | await computeNext(state, reducers, fact) 29 | } 30 | 31 | return { 32 | state, 33 | last: fact ? fact._id : lastFact 34 | } 35 | } 36 | 37 | function computeNext (state, reducers, fact) { 38 | var matched = [] 39 | var errors = [] 40 | 41 | let timestamp = parseInt(fact._id.split(':')[1]) 42 | 43 | for (let j = 0; j < reducers.length; j++) { 44 | let {rule, lineParser, fn} = reducers[j] 45 | let {status: succeeded, value: params} = lineParser.parse(fact.line) 46 | if (succeeded) { 47 | matched.push(rule) 48 | let err = tryRun(fn, [state, params, timestamp]) 49 | if (err) { 50 | errors.push(err) 51 | } 52 | } 53 | } 54 | 55 | return { 56 | state, 57 | matched, 58 | errors 59 | } 60 | } 61 | 62 | async function next (state, reducers, fact) { 63 | let before = jsondiffpatch.clone(state) 64 | let res = computeNext(state, reducers, fact) 65 | res.diff = jsondiffpatch.diff(before, res.state) 66 | return res 67 | } 68 | 69 | async function reducers () { 70 | let rules = await listRules() 71 | 72 | // compile rules 73 | var reducers = [] 74 | for (let i = 0; i < rules.length; i++) { 75 | let rule = rules[i] 76 | 77 | let { 78 | value: directives, 79 | status: ok, 80 | expected, 81 | index 82 | } = patternParser.parse(rule.pattern) 83 | if (!ok) { 84 | console.log(`error parsing pattern '${rule.pattern}: 85 | expected '${expected.join(', ')}' at index ${index.offset} 86 | but instead got '${rule.pattern[index.offset]}' (${rule.pattern.slice(index.offset - 2, index.offset + 2)})`) 87 | continue 88 | } 89 | let lineParser = makeLineParser(directives) 90 | 91 | var fn 92 | switch (rule.kind) { 93 | case 'js': 94 | fn = tryEvalRuleCode(rule.code) 95 | break 96 | default: 97 | fn = () => {} 98 | } 99 | 100 | reducers.push({ 101 | rule, 102 | lineParser, 103 | fn 104 | }) 105 | } 106 | 107 | return reducers 108 | } 109 | 110 | function tryEvalRuleCode (code) { 111 | var module = {exports: () => {}} 112 | try { 113 | eval(code) 114 | } catch (e) {} 115 | return module.exports 116 | } 117 | 118 | function tryRun (fn, args) { 119 | try { 120 | fn.apply(null, args) 121 | } catch (e) { 122 | return e 123 | } 124 | } 125 | 126 | module.exports = compute 127 | -------------------------------------------------------------------------------- /core/db.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const PouchDB = require('pouchdb-core') 3 | .plugin(require('pouchdb-adapter-leveldb')) 4 | .plugin(require('pouchdb-adapter-http')) 5 | .plugin(require('pouchdb-replication')) 6 | 7 | const file = path.join(process.cwd(), '_database') 8 | 9 | module.exports = new PouchDB(file) 10 | -------------------------------------------------------------------------------- /core/facts.js: -------------------------------------------------------------------------------- 1 | const db = require('./db') 2 | 3 | module.exports.listFacts = listFacts 4 | async function listFacts (startkey, endkey) { 5 | startkey = startkey || 'f:' 6 | endkey = endkey || 'f:~' 7 | 8 | let res = await db.allDocs({ 9 | startkey, 10 | endkey, 11 | include_docs: true 12 | }) 13 | return res.rows.map(r => r.doc) 14 | } 15 | 16 | module.exports.addFact = addFact 17 | async function addFact (line, time) { 18 | time = time || new Date() 19 | if (isNaN(time.getDate())) { 20 | throw new Error('date is invalid.') 21 | } 22 | 23 | var doc = { 24 | _id: `f:${parseInt(time.getTime() / 1000)}`, 25 | line 26 | } 27 | let res = db.put(doc) 28 | doc._rev = res.rev 29 | return doc 30 | } 31 | 32 | module.exports.fetchFact = fetchFact 33 | async function fetchFact (id) { return db.get(id) } 34 | 35 | module.exports.updateFact = updateFact 36 | async function updateFact (fact) { return db.put(fact) } 37 | 38 | module.exports.delFact = delFact 39 | async function delFact (fact) { return db.remove(fact) } 40 | -------------------------------------------------------------------------------- /core/helpers/date.js: -------------------------------------------------------------------------------- 1 | const months = [ 2 | 'Jan', 3 | 'Feb', 4 | 'Mar', 5 | 'Apr', 6 | 'May', 7 | 'Jun', 8 | 'Jul', 9 | 'Aug', 10 | 'Sep', 11 | 'Oct', 12 | 'Nov', 13 | 'Dec' 14 | ] 15 | 16 | const pad = v => require('left-pad')(v, 2, '0') 17 | 18 | const today = new Date() 19 | 20 | module.exports.formatId = function (_id) { 21 | let timestamp = parseInt(_id.split(':')[1]) 22 | if (timestamp < 100) { 23 | return `time ${timestamp}` 24 | } 25 | return module.exports.formatTimestamp(timestamp) 26 | } 27 | 28 | module.exports.formatTimestamp = function (timestamp) { 29 | let date = new Date(timestamp * 1000) 30 | 31 | if (today.getFullYear() === date.getFullYear()) { 32 | if (today.getMonth() === date.getMonth()) { 33 | if (today.getDate() === date.getDate()) { 34 | // today 35 | return `today, ${date.getHours()}:${pad(date.getMinutes())}` 36 | } else { 37 | // a different day in the same month 38 | return `${months[date.getMonth()]} ${pad(date.getDate())}, ${pad(date.getHours())}:${pad(date.getMinutes())}` 39 | } 40 | } else { 41 | // a different month in the same year 42 | return `${months[date.getMonth()]} ${pad(date.getDate())}` 43 | } 44 | } else { 45 | // a different year 46 | return `${months[date.getMonth()]} ${pad(date.getDate())} ${date.getFullYear()}` 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /core/helpers/unicode-letter.js: -------------------------------------------------------------------------------- 1 | module.exports = '[A-Za-z\\xAA\\xBA\\xC0-\\xD6\\xD8-\\xF6\\xF8-\\u02B8\\u02E0-\\u02E4\\u1D00-\\u1D25\\u1D2C-\\u1D5C\\u1D62-\\u1D65\\u1D6B-\\u1D77\\u1D79-\\u1DBE\\u1E00-\\u1EFF\\u2071\\u207F\\u2090-\\u209C\\u212A\\u212B\\u2132\\u214E\\u2160-\\u2188\\u2C60-\\u2C7F\\uA722-\\uA787\\uA78B-\\uA7AE\\uA7B0-\\uA7B7\\uA7F7-\\uA7FF\\uAB30-\\uAB5A\\uAB5C-\\uAB64\\uFB00-\\uFB06\\uFF21-\\uFF3A\\uFF41-\\uFF5A]' 2 | -------------------------------------------------------------------------------- /core/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pf-core", 3 | "version": "0.1.1", 4 | "description": "", 5 | "license": "ISC", 6 | "dependencies": { 7 | "cuid": "^1.3.8", 8 | "debug": "^3.0.0", 9 | "jsondiffpatch": "^0.2.4", 10 | "left-pad": "^1.1.3", 11 | "parsimmon": "^1.6.2", 12 | "pouchdb-adapter-http": "^6.3.4", 13 | "pouchdb-adapter-leveldb": "^6.3.4", 14 | "pouchdb-core": "^6.3.4", 15 | "pouchdb-replication": "^6.3.4" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /core/parser-parser-test.js: -------------------------------------------------------------------------------- 1 | const patternParser = require('./parser-parser') 2 | const {makeLineParser} = require('./parser') 3 | const tape = require('tape') 4 | 5 | tape('parsing user-defined rule definitions', t => { 6 | var rule = ' banana[^@boat] (de|a)' 7 | t.deepEqual(patternParser.parse(rule), { 8 | value: [ 9 | {kind: 'parameter', name: 'word', type: 'word', multiple: false}, 10 | {kind: 'whitespace'}, 11 | {kind: 'literal', string: 'banana'}, 12 | {kind: 'optional', alternatives: [ 13 | [{kind: 'literal', string: '^@boat'}] 14 | ]}, 15 | {kind: 'whitespace'}, 16 | {kind: 'alternatives', alternatives: [ 17 | [{kind: 'literal', string: 'de'}], 18 | [{kind: 'literal', string: 'a'}] 19 | ]} 20 | ], 21 | status: true 22 | }, rule) 23 | 24 | rule = 'pagamentos: [em ]' 25 | t.deepEqual(patternParser.parse(rule), { 26 | value: [ 27 | {kind: 'literal', string: 'pagamentos:'}, 28 | {kind: 'whitespace'}, 29 | {kind: 'parameter', name: 'pagamentos', type: 'money', multiple: true}, 30 | {kind: 'whitespace'}, 31 | {kind: 'optional', alternatives: [ 32 | [ 33 | {kind: 'literal', string: 'em'}, 34 | {kind: 'whitespace'}, 35 | {kind: 'parameter', name: 'date', type: 'date', multiple: false} 36 | ] 37 | ]} 38 | ], 39 | status: true 40 | }, rule) 41 | 42 | t.end() 43 | }) 44 | 45 | tape('parsing a line', t => { 46 | var parser = makeLineParser([ 47 | {kind: 'parameter', name: 'xu', type: 'word'}, 48 | {kind: 'whitespace'}, 49 | {kind: 'literal', string: 'banana'}, 50 | {kind: 'optional', alternatives: [ 51 | [{kind: 'literal', string: '-boat'}] 52 | ]} 53 | ]) 54 | 55 | var line = 'açaí banana-boat' 56 | t.deepEqual(parser.tryParse(line), {xu: 'açaí'}, line) 57 | line = 'açaí banana' 58 | t.deepEqual(parser.tryParse(line), {xu: 'açaí'}, line) 59 | 60 | parser = makeLineParser([ 61 | {kind: 'literal', string: 'pag'}, 62 | {kind: 'optional', alternatives: [ 63 | [{kind: 'literal', string: 'agamento'}], 64 | [{kind: 'literal', string: 'ou'}], 65 | [{kind: 'literal', string: 'o'}] 66 | ]}, 67 | {kind: 'whitespace'}, 68 | {kind: 'parameter', name: 'habitante', type: 'words'}, 69 | {kind: 'whitespace'}, 70 | {kind: 'parameter', name: 'valor', type: 'money'}, 71 | {kind: 'whitespace'}, 72 | {kind: 'literal', string: 'em'}, 73 | {kind: 'whitespace'}, 74 | {kind: 'parameter', type: 'date', name: 'date'} 75 | ]) 76 | 77 | line = 'pag maria euzébia 525,30 em 13/12/2018' 78 | t.deepEqual(parser.tryParse(line), {habitante: 'maria euzébia', valor: 525.30, date: '2018-12-13'}, line) 79 | line = 'pagou joana francisca 725,30 em 18/01/2019' 80 | t.deepEqual(parser.tryParse(line), {habitante: 'joana francisca', valor: 725.30, date: '2019-01-18'}, line) 81 | 82 | parser = makeLineParser([ 83 | {kind: 'alternatives', alternatives: [ 84 | [ 85 | {kind: 'literal', string: 'débito:'}, 86 | {kind: 'whitespace'}, 87 | {kind: 'parameter', name: 'débito', type: 'money', multiple: true} 88 | ], 89 | [ 90 | {kind: 'literal', string: 'crédito:'}, 91 | {kind: 'whitespace'}, 92 | {kind: 'parameter', name: 'crédito', type: 'money', multiple: true} 93 | ] 94 | ]} 95 | ]) 96 | 97 | line = 'débito: 18, 25,40' 98 | t.deepEqual(parser.tryParse(line), {'débito': [18, 25.40]}, line) 99 | line = 'crédito: 38.16' 100 | t.deepEqual(parser.tryParse(line), {'crédito': [38.16]}, line) 101 | 102 | parser = makeLineParser([ 103 | {kind: 'parameter', name: 'nome', type: 'words'}, 104 | {kind: 'whitespace'}, 105 | {kind: 'literal', string: 'pagou'} 106 | ]) 107 | 108 | line = 'fulano pagou' 109 | t.deepEqual(parser.tryParse(line), {nome: 'fulano'}, line) 110 | line = 'fulano de tal pagou' 111 | t.deepEqual(parser.tryParse(line), {nome: 'fulano de tal'}, line) 112 | 113 | t.end() 114 | }) 115 | 116 | tape('both things', t => { 117 | t.deepEqual( 118 | makeLineParser( 119 | patternParser.parse( 120 | 'pac[iente] [da|do] dr[a][.] pag[ou|.][:] [dia ]' 121 | ).value 122 | ).tryParse('paciente beltrano armando da dra. mariana gastón pagou: 600 dia 18/12/2001'), { 123 | dent: 'mariana gastón', 124 | pac: 'beltrano armando', 125 | money: 600, 126 | date: '2001-12-18' 127 | } 128 | ) 129 | 130 | t.deepEqual( 131 | makeLineParser( 132 | patternParser.parse( 133 | ' [has] paid [on ]' 134 | ).value 135 | ).tryParse('someone else has paid 12 on 23/11/2019'), { 136 | someone: 'someone else', 137 | money: 12, 138 | date: '2019-11-23' 139 | } 140 | ) 141 | 142 | t.deepEqual( 143 | makeLineParser( 144 | patternParser.parse( 145 | ' [has] paid [on ]' 146 | ).value 147 | ).tryParse('someone else paid 12 on 23/11/2019'), { 148 | someone: 'someone else', 149 | money: 12, 150 | date: '2019-11-23' 151 | } 152 | ) 153 | 154 | t.deepEqual( 155 | makeLineParser( 156 | patternParser.parse( 157 | 'chegou ' 158 | ).value 159 | ).tryParse('chegou fulano'), { 160 | someone: 'fulano' 161 | } 162 | ) 163 | 164 | 165 | let today = new Date() 166 | today.setDate(15) 167 | t.deepEqual( 168 | makeLineParser( 169 | patternParser.parse( 170 | ' [do ] pagou [dia ]' 171 | ).value 172 | ).tryParse('Maria Angélica do C2 pagou 777,40 dia 15'), { 173 | nome: 'Maria Angélica', 174 | quarto: 'C2', 175 | valor: 777.40, 176 | date: today.toISOString().split('T')[0] 177 | } 178 | ) 179 | 180 | t.end() 181 | }) 182 | -------------------------------------------------------------------------------- /core/parser-parser.js: -------------------------------------------------------------------------------- 1 | const P = require('parsimmon') 2 | 3 | const unicodeLetter = require('./helpers/unicode-letter') 4 | 5 | const lt = P.string('<') 6 | const gt = P.string('>') 7 | const lsquare = P.string('[') 8 | const rsquare = P.string(']') 9 | const lbracket = P.string('(') 10 | const rbracket = P.string(')') 11 | const ellipsis = P.alt(P.string('…'), P.string('...')) 12 | const colon = P.string(':') 13 | const vertical = P.string('|') 14 | const word = P.regexp(new RegExp(`${unicodeLetter}+`)) 15 | 16 | const directive = P.alt( 17 | P.string('words'), 18 | P.string('word'), 19 | P.string('numberword'), 20 | P.string('money'), 21 | P.string('date'), 22 | P.string('number') 23 | ) 24 | 25 | const parameter = P.seq( 26 | lt, 27 | P.seq(word, colon).atMost(1).map(([wc]) => wc && wc[0]), 28 | directive, 29 | ellipsis.atMost(1).map(([elp]) => Boolean(elp)), 30 | gt 31 | ).map(([_, name, directive, multiple]) => ({ 32 | kind: 'parameter', 33 | name: name || directive, 34 | type: directive, 35 | multiple 36 | })) 37 | 38 | const optional = P.seq( 39 | lsquare, 40 | P.lazy(() => main).sepBy1(vertical), 41 | rsquare 42 | ).map(([_, alternatives]) => ({ 43 | kind: 'optional', 44 | alternatives 45 | })) 46 | 47 | const alternatives = P.seq( 48 | lbracket, 49 | P.lazy(() => main).sepBy1(vertical), 50 | rbracket 51 | ).map(([_, alternatives]) => ({ 52 | kind: 'alternatives', 53 | alternatives 54 | })) 55 | 56 | const literal = P.noneOf('[<()>]| ') 57 | .atLeast(1) 58 | .map(results => ({kind: 'literal', string: results.join('')})) 59 | 60 | var main = P.alt( 61 | P.whitespace.result({kind: 'whitespace'}), 62 | parameter, 63 | optional, 64 | alternatives, 65 | literal 66 | ).atLeast(1) 67 | 68 | module.exports = { 69 | parse (pattern) { 70 | return main.parse(pattern.trim()) 71 | }, 72 | tryParse (pattern) { 73 | return main.tryParse(pattern.trim()) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /core/parser.js: -------------------------------------------------------------------------------- 1 | const P = require('parsimmon') 2 | const xtend = require('xtend') 3 | 4 | const unicodeLetter = require('./helpers/unicode-letter') 5 | 6 | module.exports.makeLineParser = makeLineParser 7 | 8 | function makeLineParser (directives) { 9 | return P.seq.apply(P, directives.map((directive, i) => 10 | parserFromDirective(directive, i, directives) 11 | )) 12 | .map(args => args 13 | .filter(x => typeof x === 'object' && !Array.isArray(x)) 14 | .reduce((acc, elem) => xtend(acc, elem), {}) 15 | ) 16 | } 17 | 18 | function parserFromDirective (directive, i, directives) { 19 | switch (directive.kind) { 20 | case 'whitespace': 21 | return P.optWhitespace 22 | case 'literal': 23 | return P.string(directive.string) 24 | case 'alternatives': 25 | return P.alt.apply(P, directive.alternatives.map(makeLineParser)) 26 | case 'optional': 27 | return P.alt.apply(P, directive.alternatives.map(makeLineParser)) 28 | .or(P.optWhitespace) 29 | case 'parameter': 30 | return P.lazy(() => { 31 | if (directive.type === 'words') { 32 | let following = directives.slice(i + 1) 33 | return wordsUntil(P.seq.apply(P, following.map(parserFromDirective)), []) 34 | } 35 | 36 | let typeParser = types[directive.type] 37 | if (directive.multiple) { 38 | typeParser = P.sepBy1( 39 | typeParser, 40 | P.seq( 41 | P.optWhitespace, 42 | P.string(directive.separator || ','), 43 | P.optWhitespace 44 | ) 45 | ) 46 | } 47 | return typeParser 48 | }) 49 | .map(v => ({[directive.name]: v})) 50 | } 51 | } 52 | 53 | const word = P.regexp(new RegExp(`${unicodeLetter}+`)) 54 | .desc('_a word_') 55 | 56 | const wordsUntil = (nextParser, words) => 57 | word.skip(P.optWhitespace) 58 | .chain(w => { 59 | let nwords = words.concat(w) 60 | return P.lookahead(nextParser) 61 | .map(() => nwords.join(' ')) 62 | .or(P.lazy(() => wordsUntil(nextParser, nwords))) 63 | }) 64 | 65 | const numberword = P.regexp(new RegExp(`(?:\\d|${unicodeLetter})+`)) 66 | .desc('_a word mingled with numbers_') 67 | 68 | const day = P.regexp(/(?:[0-2]\d|3[0-1]|\d)/) 69 | .desc('_a day (like 7, 18 or 31) _') 70 | .map(x => { 71 | let today = new Date() 72 | today.setDate(parseInt(x)) 73 | return today.toISOString().split('T')[0] 74 | }) 75 | 76 | const date = P.alt( 77 | P.regexp(/\d{1,2}\/\d{1,2}\/\d{4}/).map(x => x.split('/').reverse().join('-')), 78 | P.regexp(/\d{1,2}\/\d{1,2}/).map(x => { 79 | let today = new Date() 80 | let [day, month] = x.split('/') 81 | today.setDate(parseInt(day)) 82 | today.setMonth(parseInt(month) - 1) 83 | return today.toISOString().split('T')[0] 84 | }), 85 | day 86 | ) 87 | .desc('_date_') 88 | 89 | const decimal = P.regexp(/\d+(?:[,.]\d{2})?/) 90 | .desc('_number optionally with decimals_') 91 | .map(x => parseFloat(x.replace(',', '.'))) 92 | 93 | const integer = P.regexp(/\d+/) 94 | .desc('_number without decimals_') 95 | .map(x => parseInt(x)) 96 | 97 | const money = P.seq( 98 | P.regexp(/(\$|\$ )?/), 99 | decimal 100 | ) 101 | .map(([_, n]) => n) 102 | 103 | const types = { 104 | word, 105 | day, 106 | date, 107 | money, 108 | number: integer, 109 | integer, 110 | decimal, 111 | numberword 112 | } 113 | -------------------------------------------------------------------------------- /core/rules.js: -------------------------------------------------------------------------------- 1 | const cuid = require('cuid') 2 | 3 | const db = require('./db') 4 | 5 | module.exports.addRule = addRule 6 | async function addRule (kind, pattern, code) { 7 | return db.put({ 8 | _id: `r:${cuid.slug()}`, 9 | kind, 10 | pattern, 11 | code 12 | }) 13 | } 14 | 15 | module.exports.listRules = listRules 16 | async function listRules () { 17 | try { 18 | let res = await db.allDocs({ 19 | startkey: 'r:', 20 | endkey: 'r:~', 21 | include_docs: true 22 | }) 23 | return res.rows.map(r => r.doc) 24 | } catch (e) { 25 | console.log(e) 26 | } 27 | } 28 | 29 | module.exports.fetchRule = fetchRule 30 | async function fetchRule (id) { return db.get(id) } 31 | 32 | module.exports.updateRule = updateRule 33 | async function updateRule (rule) { return db.put(rule) } 34 | 35 | module.exports.delRule = delRule 36 | async function delRule (rule) { return db.remove(rule) } 37 | -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "lerna": "2.0.0", 3 | "packages": [ 4 | "core", 5 | "cli" 6 | ], 7 | "version": "0.1.1" 8 | } 9 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pf-cli", 3 | "version": "0.1.0", 4 | "description": "prato feito", 5 | "repository": { 6 | "type": "git", 7 | "url": "git+https://github.com/fiatjaf/pf.git" 8 | }, 9 | "author": "fiatjaf@gmail.com", 10 | "license": "ISC", 11 | "devDependencies": { 12 | "lerna": "^2.0.0", 13 | "tape": "^4.7.0" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /screencast.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fiatjaf/pf/d88ea58f6724413cf7e86fee456705383ee363b2/screencast.gif --------------------------------------------------------------------------------