├── .gitignore ├── .npmignore ├── LICENSE ├── README.md ├── index.js ├── package-lock.json ├── package.json └── src ├── EntityService.js ├── GuessService.js ├── SampleService.js ├── Wit.js └── lib ├── Entity.js ├── Guess.js ├── Language.js └── init.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .idea 3 | run.js 4 | npm-debug.log 5 | .nyc_output 6 | backup.zip 7 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | test/ 2 | 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018, Hexastack 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 1. Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | 2. Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | 3. All advertising materials mentioning features or use of this software 12 | must display the following acknowledgement: 13 | This product includes software developed by Hexastack. 14 | 4. Neither the name of Hexastack nor the 15 | names of its contributors may be used to endorse or promote products 16 | derived from this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER ''AS IS'' AND ANY 19 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY 22 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # wit-api 2 | wit-api is a node package that allows you to easily configure, train and consume your NLP through Wit.ai HTTP API. 3 | 4 | **Note**: This is the version 1.x.y, it has no dependency, and it uses `promises`. 5 | 6 | **Note**: Version 1.0.0 is stable, and will recieve only additional utils methods 7 | 8 | ## Installation 9 | ```bash 10 | npm i wit-api --save 11 | ``` 12 | 13 | ## Initialization 14 | ```javascript 15 | const Wit = require('./src/Wit') 16 | 17 | async function run () { 18 | try { 19 | // Instantiation 20 | const wit = new Wit('') 21 | // Syncronize with remote 22 | await wit.sync() 23 | } catch (e) { 24 | console.error(e) 25 | } 26 | } 27 | 28 | run() 29 | ``` 30 | Additionally, you can pass options to the instantiation, like: 31 | `new Wit('', { debug: true, timeout: 30000, version: '20170307'})` 32 | 33 | - debug: will display more insights about the http requests 34 | - timeout: in milliseconds, indicate when the http request times-out 35 | - version: indicate what version of wit.ai to use, **this lib ain't work for versions prior to 20170307** 36 | 37 | ## Entities, Vlaues & Expressions 38 | ```javascript 39 | async function run () { 40 | try { 41 | const wit = new Wit('') 42 | await wit.sync() 43 | // Adding a new entity 44 | await wit.entities.add('cloth') 45 | // The newly added entity is sync-ed in the `wit` object 46 | // and can be accessed through `wit.entities.cloth` 47 | 48 | // For script reusability, it is recomended check the entity existance before adding it 49 | if (!wit.entities.size) { 50 | await wit.entities.add('size') 51 | } 52 | 53 | // Update an entity 54 | await wit.entities.cloth.update({ doc: 'An item from the store', lookups: ['keywords'] }) 55 | 56 | // We can also manipulate the entities values and expression through the `.update` function 57 | // Also additional options are present, like metadata 58 | await wit.entities.size.update({ 59 | doc: 'Size', 60 | lookups: ['free-text', 'keywords'], 61 | values: [ 62 | {value: 'extra small', expressions: ['36'], metadata: 'XS'}, 63 | {value: 'small', expressions: ['37', '38'], metadata: 'S'}, 64 | {value: 'medium', expressions: ['39', '40'], metadata: 'M'}, 65 | {value: 'large', expressions: ['41', '42'], metadata: 'L'}, 66 | {value: 'extra large', expressions: ['43', '44'], metadata: 'XL'} 67 | ] 68 | }) 69 | 70 | // Deleting an entity 71 | await wit.entities.cloth.destroy() 72 | } catch (e) { 73 | console.error(e) 74 | } 75 | } 76 | 77 | run() 78 | ``` 79 | ## Training 80 | ```javascript 81 | // This goes insde an `async function` 82 | await wit.services.sample.train([ 83 | { 84 | text: 'I would like to have a jean please.', 85 | entities: [ 86 | { 87 | entity: 'intent', 88 | value: 'order' 89 | }, 90 | { 91 | entity: 'product', 92 | value 'pant', 93 | start: '22', 94 | end: '26' 95 | } 96 | ] 97 | } 98 | ]) 99 | // wit.service.sample.train can take multiples training sample 100 | 101 | // You can reverse a training by using .forget 102 | await wit.services.sample.forget([ 103 | {text: 'I would like to have a jean please.'} 104 | ]) 105 | ``` 106 | ## Guessing (text & speech) 107 | This is actually where we use the api for understanding 108 | 109 | All the guess methods retuen a Guess Object 110 | ```javascript 111 | // this goes inside an `async function` 112 | 113 | // Text: 114 | const guess = await wit.services.guess.message('I want to try a coton black shirt, preferably size 43') 115 | console.log(guess) 116 | /* example of a result: 117 | { _text: 'I want to try a coton black shirt, preferably size 43', 118 | entities: 119 | { product: [ { confidence: 1, value: 'shirt', type: 'value' } ], 120 | shirtTextile: [ { confidence: 1, value: 'coton', type: 'value' } ], 121 | color: 122 | [ { metadata: '#000000', 123 | confidence: 1, 124 | value: 'black', 125 | type: 'value' } ], 126 | size: [ { confidence: 1, value: 'XL', type: 'value' } ], 127 | intent: [ { confidence: 0.72456770482335, value: 'try' } ] }, 128 | msg_id: '1AY90y2ewdQhotazg' 129 | } 130 | */ 131 | 132 | // Language: 133 | await wit.services.guess.language('je m\'appel foobar') 134 | 135 | // You can also have both language guessing and text guessing by using 136 | await wit.services.guess.all('I want to try a coton black shirt, preferably size 43') 137 | 138 | // Audio: 139 | await wit.services.guess.speech('path/to/an/audio/file.wav') 140 | ``` 141 | Both `.message` and `.speech` can take additional options, in like the following 142 | ```javascript 143 | const options = {n: 5} 144 | await wit.services.guess.message(text, options) 145 | ``` 146 | More about these options at https://wit.ai/docs/http/20170307#get__message_link 147 | 148 | ## Guess Methods 149 | Once the guessing service was called, it returns a Guess Object, theses objects have utils methods. 150 | 151 | For instance to return only relevant guesses (with confidance above 0.95) 152 | ```javascript 153 | const guess = await wit.services.guess.all('I want to try a coton black shirt, preferably size 43', {n: 5}) 154 | console.log(guess.bestGuesses(.95)) 155 | /* 156 | would print: 157 | { 158 | "shirtTextile": {"confidence": 1, "value": "coton", "type": "value"}, 159 | "color": {"metadata": "#000000", "confidence": 1, "value": "black", "type": "value"}, 160 | "product": {"confidence": 1, "value": "shirt", "type": "value"}, 161 | "size": {"confidence": 1, "value": "XL", "type": "value"}, 162 | "intent": {"confidence": 0.98808266775175, "value": "request"}, 163 | "language": {"locale": "en_XX", "confidence": 1} 164 | } 165 | */ 166 | 167 | // Additionally you can filter and softfilter guessing results using: 168 | guess.filter(.95) 169 | guess.softFilter(1) 170 | ``` 171 | 172 | ###### with love from Hexastack 173 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const Wit = require('./src/Wit') 2 | module.exports = Wit 3 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "wit-api", 3 | "version": "1.0.1", 4 | "lockfileVersion": 1 5 | } 6 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "wit-api", 3 | "version": "1.1.0", 4 | "description": "Implementation of wit api, provides methods to easily manage your wit app", 5 | "main": "index.js", 6 | "keywords": [ 7 | "wit", 8 | "api", 9 | "nlp", 10 | "wit-api", 11 | "ai", 12 | "bot", 13 | "chatbot", 14 | "ES6", 15 | "async/await" 16 | ], 17 | "publishConfig": { 18 | "registry": "https://npm.pkg.github.com/" 19 | }, 20 | "scripts": { 21 | "test": "echo 'no test exists -FOR NOW-'" 22 | }, 23 | "repository": { 24 | "type": "git", 25 | "url": "https://github.com/Hexastack/wit-api.git" 26 | }, 27 | "author": "Hexastack ", 28 | "license": "BSD-4-Clause" 29 | } 30 | -------------------------------------------------------------------------------- /src/EntityService.js: -------------------------------------------------------------------------------- 1 | const actions = function (id) { 2 | return { 3 | get: { 4 | method: 'GET', 5 | path: '/entities' 6 | }, 7 | find: { 8 | method: 'GET', 9 | path: `/entities/${id}` 10 | }, 11 | add: { 12 | method: 'POST', 13 | path: '/entities' 14 | }, 15 | update: { 16 | method: 'PUT', 17 | path: `/entities/${id}` 18 | }, 19 | delete: { 20 | method: 'DELETE', 21 | path: `/entities/${id}` 22 | } 23 | } 24 | } 25 | 26 | module.exports = function (request) { 27 | return { 28 | async get () { 29 | const options = actions().get 30 | return await request(options) 31 | }, 32 | async find (id) { 33 | const options = actions(id).find 34 | return await request(options) 35 | }, 36 | async add (name, doc = '') { 37 | const options = actions().add 38 | return await request(options, {id: name, doc}) 39 | }, 40 | async update (id, changes) { 41 | const options = actions(id).update 42 | return await request(options, changes) 43 | }, 44 | async delete (id) { 45 | const options = actions(id).delete 46 | return await request(options) 47 | }, 48 | async sync () { 49 | try { 50 | let entities = await this.get() 51 | return await Promise.all(entities.map(e => this.find(e))) 52 | } catch(e) { 53 | console.error(e) 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/GuessService.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | const path = require('path') 3 | const Guess = require('./lib/Guess') 4 | const Language = require('./lib/Language') 5 | 6 | const actions = function (params, type = 'wav', format = {}) { 7 | format = Object.assign({ encoding: 'unsigned-integer', bits: 8, rate: 8000, endian: 'little' }, format) 8 | let headers = { 9 | 'Content-Type': `audio/${type}`, 10 | 'Transfer-encoding': 'chunked' 11 | } 12 | if (type === 'raw') { 13 | headers['content-type'] = `audio/${type};encoding=${format.encoding};bits=${format.bits};rate=${format.rate};endian=${format.endian}` 14 | } 15 | 16 | return { 17 | message: { 18 | method: 'GET', 19 | path: '/message', 20 | qs: params 21 | }, 22 | language: { 23 | method: 'GET', 24 | path: '/language', 25 | qs: params 26 | }, 27 | speech: { 28 | method: 'POST', 29 | path: '/speech', 30 | qs: params, 31 | headers 32 | } 33 | } 34 | } 35 | 36 | const readfile = function (filePath) { 37 | let type = path.extname(filePath).replace(/^\./, '') 38 | if (type === 'mp3') { 39 | type = 'mpeg3' 40 | } 41 | return { type, audio: fs.readFileSync(filePath) } 42 | } 43 | 44 | module.exports = function (request) { 45 | return { 46 | async message(text, params = {}) { 47 | params.q = text 48 | const options = actions(params).message 49 | return new Guess(await request(options)) 50 | }, 51 | async language(text, params = {}) { 52 | params.q = text 53 | const options = actions(params).language 54 | return new Language(await request(options)) 55 | }, 56 | async all(text, params = {}) { 57 | params.q = text 58 | const msgOptions = actions(params).message 59 | const langOptions = actions(params).language 60 | const responses = await Promise.all([request(msgOptions), request(langOptions)]) 61 | return new Guess(responses[0], new Language(responses[1])) 62 | }, 63 | async speech(filePath, params, format) { 64 | const { type, audio } = readfile(filePath) 65 | const options = actions(params, type, format).speech 66 | return new Guess(await request(options, audio)) 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/SampleService.js: -------------------------------------------------------------------------------- 1 | const actions = function () { 2 | return { 3 | get: { 4 | method: 'GET', 5 | path: '/samples' 6 | }, 7 | add: { 8 | method: 'POST', 9 | path: '/samples' 10 | }, 11 | delete: { 12 | method: 'DELETE', 13 | path: '/samples' 14 | } 15 | } 16 | } 17 | 18 | module.exports = function (request) { 19 | return { 20 | async get (limit = 10, offset = 0, entities = [], values = [], isNegative = false) { 21 | let options = actions().get 22 | options.qs = {limit, offset, negative: isNegative} 23 | if (entities.length) { 24 | options.qs.entity_ids = entities 25 | if (values.length) { 26 | options.qs.entity_values = values 27 | } 28 | } 29 | 30 | return await request(options) 31 | }, 32 | async add (text, entities) { 33 | const options = actions().add 34 | return await request(options, [{text, entities}]) 35 | }, 36 | async train (dataset) { 37 | const options = actions().add 38 | return await request(options, dataset) 39 | }, 40 | async delete (text) { 41 | const options = actions().delete 42 | return await request(options, [{text}]) 43 | }, 44 | async forget (dataset) { 45 | const options = action().delete 46 | return await request(options, dataset.map(d => { 47 | return {text: d.text} 48 | })) 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/Wit.js: -------------------------------------------------------------------------------- 1 | const init = require('./lib/init') 2 | const SampleService = require('./SampleService') 3 | const EntityService = require('./EntityService') 4 | const GuessService = require('./GuessService') 5 | const Entity = require('./lib/Entity') 6 | 7 | Date.prototype.getFormattedDate = function () { 8 | const date = this.getDate() 9 | return date < 10 ? `0${date}` : date 10 | } 11 | Date.prototype.getFormattedMonth = function () { 12 | const month = this.getMonth() + 1 13 | return month < 10 ? `0${month}` : month 14 | } 15 | 16 | const getVersion = function () { 17 | const date = new Date() 18 | return `${date.getFullYear()}${date.getFormattedMonth()}${date.getFormattedDate()}` 19 | } 20 | 21 | const Wit = function (token, { version, timeout, debug}) { 22 | this.request = init(token, version || getVersion(), timeout || 20000, debug) 23 | this.services = { 24 | sample: SampleService(this.request), 25 | entity: EntityService(this.request), 26 | guess: GuessService(this.request) 27 | } 28 | this.entities = {} 29 | this.reset = () => { 30 | this.entities = {} 31 | this.entities.add = async (name, doc) => { 32 | try { 33 | const entity = await this.services.entity.add(name, doc) 34 | this.entities[entity.name] = new Entity(this.services.entity, entity, this.entities) 35 | } catch (e) { 36 | console.error(e) 37 | } 38 | } 39 | } 40 | this.reset() 41 | this.init = async () => { 42 | try { 43 | const entities = await this.services.entity.sync() 44 | entities.forEach(e => { 45 | this.entities[e.name] = new Entity(this.services.entity, e, this.entities) 46 | }) 47 | } catch (e) { 48 | console.error(e) 49 | } 50 | } 51 | this.sync = async () => { 52 | this.reset() 53 | await this.init() 54 | } 55 | } 56 | 57 | module.exports = Wit 58 | -------------------------------------------------------------------------------- /src/lib/Entity.js: -------------------------------------------------------------------------------- 1 | const Entity = function (service, entity, parent) { 2 | this.service = service 3 | this.id = entity.id 4 | this.name = entity.name 5 | this.lang = entity.lang || 'en' 6 | this.doc = entity.doc || '' 7 | this.lookups = entity.lookups || [] 8 | this.values = entity.values || [] 9 | this.builtin = entity.builtin || false 10 | this.parent = parent 11 | this.update = async (changes) => { 12 | try { 13 | const entity = await this.service.update(this.name, changes) 14 | this.parent[this.name] = new Entity(this.service, entity, this.parent) 15 | } catch (e) { 16 | console.error(e) 17 | } 18 | } 19 | this.destroy = async () => { 20 | try { 21 | const id = await this.service.delete(this.name) 22 | if (id) 23 | return delete this.parent[this.name] 24 | throw (new Error('already_deleted')) 25 | } catch (e) { 26 | if (e.message === 'already_deleted') { 27 | console.warn(e) 28 | } else { 29 | console.error(e) 30 | } 31 | } 32 | } 33 | } 34 | 35 | module.exports = Entity 36 | -------------------------------------------------------------------------------- /src/lib/Guess.js: -------------------------------------------------------------------------------- 1 | const Guess = function (guess, language) { 2 | this.msg_id = guess.msg_id 3 | this._text = guess._text 4 | this.entities = guess.entities 5 | this.language = language 6 | this.filter = (threshold = .9) => { 7 | const filtered = {} 8 | Object.keys(this.entities).forEach(key => { 9 | filtered[key] = this.entities[key].filter(e => e.confidence >= threshold).sort((a, b) => b.confidence - a.confidence) 10 | }) 11 | if(this.language) { 12 | filtered.language = this.language.filter(threshold) 13 | } 14 | return filtered 15 | } 16 | this.softFilter = (threshold = .9) => { 17 | const filtered = {} 18 | Object.keys(this.entities).forEach(key => { 19 | filtered[key] = this.entities[key].filter(e => e.confidence >= threshold).sort((a, b) => b.confidence - a.confidence) 20 | if (!filtered[key].length) { 21 | filtered[key] = [this.entities[key].sort((a, b) => b.confidence - a.confidence)[0]] 22 | } 23 | }) 24 | if(this.language) { 25 | filtered.language = this.language.softFilter(threshold) 26 | } 27 | return filtered 28 | } 29 | this.bestGuesses = () => { 30 | const filtered = {} 31 | Object.keys(this.entities).forEach(key => { 32 | filtered[key] = this.entities[key].sort((a, b) => b.confidence - a.confidence)[0] 33 | }) 34 | if(this.language) { 35 | filtered.language = this.language.bestGuesses() 36 | } 37 | return filtered 38 | } 39 | } 40 | 41 | module.exports = Guess 42 | -------------------------------------------------------------------------------- /src/lib/Language.js: -------------------------------------------------------------------------------- 1 | const Language = function (language) { 2 | this.detected_locales = language.detected_locales 3 | this.filter = (threshold = .9) => { 4 | return this.detected_locales.filter(e => e.confidence >= threshold).sort((a, b) => b.confidence - a.confidence) 5 | } 6 | this.softFilter = (threshold = .9) => { 7 | const filtered = this.detected_locales.filter(e => e.confidence >= threshold).sort((a, b) => b.confidence - a.confidence) 8 | if (!filtered.length) { 9 | filtered = [this.detected_locales.sort((a, b) => b.confidence - a.confidence)[0]] 10 | } 11 | return filtered 12 | } 13 | this.bestGuesses = () => { 14 | return this.detected_locales.sort((a, b) => b.confidence - a.confidence)[0] 15 | } 16 | } 17 | 18 | module.exports = Language 19 | -------------------------------------------------------------------------------- /src/lib/init.js: -------------------------------------------------------------------------------- 1 | const https = require('https') 2 | const qs = require('querystring') 3 | 4 | module.exports = init = function (token, version = '20170307', timeout = 20000, debug = false) { 5 | return function (options, data) { 6 | return new Promise((resolve, reject) => { 7 | 8 | Object.assign(options, { 9 | hostname: 'api.wit.ai', 10 | port: 443, 11 | timeout 12 | }) 13 | 14 | if (!options.headers) options.headers = {} 15 | Object.assign(options.headers, { 16 | Accept: 'application/json', 17 | Authorization: `Bearer ${token}`, 18 | Connection: 'keep-alive', 19 | 'Keep-Alive': 'timeout=10, max=1000' 20 | }) 21 | if (!options.headers['Content-Type']) options.headers['Content-Type'] = 'application/json; charset=utf-8' 22 | 23 | if (!options.qs) options.qs = {} 24 | Object.assign(options.qs, { v: version }) 25 | 26 | if (data && options.headers['Content-Type'] === 'application/json; charset=utf-8') 27 | try { 28 | data = JSON.stringify(data) 29 | } catch (e) { 30 | return reject(e) 31 | } 32 | 33 | if (options.method && ['POST', 'PUT', 'PATCH'].indexOf(options.method.toUpperCase()) !== -1) 34 | options.headers['Content-Length'] = Buffer.byteLength(data) 35 | 36 | options.path += '?' + qs.stringify(options.qs) 37 | 38 | let timeouted = false 39 | const req = https.request(options, function (res) { 40 | if (debug) 41 | console.debug(res.req._header) 42 | let body = '' 43 | res.setEncoding('utf8') 44 | res.on('data', function (chunk) { 45 | body += chunk 46 | }) 47 | res.on('end', function () { 48 | let response = {} 49 | try { 50 | response = JSON.parse(body) 51 | } catch (e) { 52 | return reject(e) 53 | } 54 | if (response.error) 55 | return reject(new Error(response.error)) 56 | return resolve(response) 57 | }) 58 | }) 59 | req.on('timeout', () => { 60 | timeouted = true 61 | req.abort() 62 | }) 63 | req.on('abort', (err) => { 64 | if (timeouted) { 65 | return reject(new Error(`Request timed-out, request was ${req._header}`)) 66 | } 67 | return reject(new Error(`Request aborted, request was ${req._header}`)) 68 | }) 69 | req.on('error', function (err) { 70 | return reject(err) 71 | }) 72 | if (options.method && ['POST', 'PUT', 'PATCH'].indexOf(options.method.toUpperCase()) !== -1) 73 | req.write(data) 74 | req.end() 75 | }) 76 | } 77 | } 78 | --------------------------------------------------------------------------------