├── .gitignore ├── .jshintrc ├── .travis.yml ├── README.md ├── lib ├── connection-pool.js ├── index.js └── types.js ├── package-lock.json ├── package.json └── test └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | node_modules/ 3 | test/index.html 4 | bundle.js 5 | .gittoken 6 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | // JSHint Default Configuration File (as on JSHint website) 3 | // See http://jshint.com/docs/ for more details 4 | 5 | "maxerr" : 50, // {int} Maximum error before stopping 6 | "esversion" : 6, 7 | 8 | // Enforcing 9 | "bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.) 10 | "camelcase" : false, // true: Identifiers must be in camelCase 11 | "curly" : true, // true: Require {} for every new block or scope 12 | "eqeqeq" : true, // true: Require triple equals (===) for comparison 13 | "forin" : false, // true: Require filtering for..in loops with obj.hasOwnProperty() 14 | "freeze" : true, // true: prohibits overwriting prototypes of native objects such as Array, Date etc. 15 | "immed" : false, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());` 16 | "indent" : 4, // {int} Number of spaces to use for indentation 17 | "latedef" : false, // true: Require variables/functions to be defined before being used 18 | "newcap" : false, // true: Require capitalization of all constructor functions e.g. `new F()` 19 | "noarg" : true, // true: Prohibit use of `arguments.caller` and `arguments.callee` 20 | "noempty" : true, // true: Prohibit use of empty blocks 21 | "nonbsp" : true, // true: Prohibit "non-breaking whitespace" characters. 22 | "nonew" : false, // true: Prohibit use of constructors for side-effects (without assignment) 23 | "plusplus" : false, // true: Prohibit use of `++` & `--` 24 | "quotmark" : false, // Quotation mark consistency: 25 | // false : do nothing (default) 26 | // true : ensure whatever is used is consistent 27 | // "single" : require single quotes 28 | // "double" : require double quotes 29 | "undef" : true, // true: Require all non-global variables to be declared (prevents global leaks) 30 | "unused" : true, // true: Require all defined variables be used 31 | "strict" : false, // true: Requires all functions run in ES5 Strict Mode 32 | "maxparams" : false, // {int} Max number of formal params allowed per function 33 | "maxdepth" : false, // {int} Max depth of nested blocks (within functions) 34 | "maxstatements" : false, // {int} Max number statements per function 35 | "maxcomplexity" : false, // {int} Max cyclomatic complexity per function 36 | "maxlen" : false, // {int} Max number of characters per line 37 | 38 | // Relaxing 39 | "asi" : true, // true: Tolerate Automatic Semicolon Insertion (no semicolons) 40 | "boss" : false, // true: Tolerate assignments where comparisons would be expected 41 | "debug" : false, // true: Allow debugger statements e.g. browser breakpoints. 42 | "eqnull" : false, // true: Tolerate use of `== null` 43 | "es5" : false, // true: Allow ES5 syntax (ex: getters and setters) 44 | "esnext" : false, // true: Allow ES.next (ES6) syntax (ex: `const`) 45 | "moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features) 46 | // (ex: `for each`, multiple try/catch, function expression…) 47 | "evil" : false, // true: Tolerate use of `eval` and `new Function()` 48 | "expr" : false, // true: Tolerate `ExpressionStatement` as Programs 49 | "funcscope" : false, // true: Tolerate defining variables inside control statements 50 | "globalstrict" : false, // true: Allow global "use strict" (also enables 'strict') 51 | "iterator" : false, // true: Tolerate using the `__iterator__` property 52 | "lastsemic" : false, // true: Tolerate omitting a semicolon for the last statement of a 1-line block 53 | "laxbreak" : false, // true: Tolerate possibly unsafe line breakings 54 | "laxcomma" : false, // true: Tolerate comma-first style coding 55 | "loopfunc" : false, // true: Tolerate functions being defined in loops 56 | "multistr" : false, // true: Tolerate multi-line strings 57 | "noyield" : false, // true: Tolerate generator functions with no yield statement in them. 58 | "notypeof" : false, // true: Tolerate invalid typeof operator values 59 | "proto" : false, // true: Tolerate using the `__proto__` property 60 | "scripturl" : false, // true: Tolerate script-targeted URLs 61 | "shadow" : false, // true: Allows re-define variables later in code e.g. `var x=1; x=2;` 62 | "sub" : true, // true: Tolerate using `[]` notation when it can still be expressed in dot notation 63 | "supernew" : false, // true: Tolerate `new function () { ... };` and `new Object;` 64 | "validthis" : false, // true: Tolerate using this in a non-constructor function 65 | 66 | // Environments 67 | "browser" : true, // Web Browser (window, document, etc) 68 | "browserify" : false, // Browserify (node.js code in the browser) 69 | "couch" : false, // CouchDB 70 | "devel" : true, // Development/debugging (alert, confirm, etc) 71 | "dojo" : false, // Dojo Toolkit 72 | "jasmine" : false, // Jasmine 73 | "jquery" : false, // jQuery 74 | "mocha" : true, // Mocha 75 | "mootools" : false, // MooTools 76 | "node" : true, // Node.js 77 | "nonstandard" : false, // Widely adopted globals (escape, unescape, etc) 78 | "prototypejs" : false, // Prototype and Scriptaculous 79 | "qunit" : false, // QUnit 80 | "rhino" : false, // Rhino 81 | "shelljs" : false, // ShellJS 82 | "worker" : false, // Web Workers 83 | "wsh" : false, // Windows Scripting Host 84 | "yui" : false, // Yahoo User Interface 85 | 86 | // Custom Globals 87 | "globals" : {} // additional predefined global variables 88 | } 89 | 90 | 91 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | 3 | language: node_js 4 | node_js: 5 | - "node" 6 | - "6" 7 | - "7" 8 | - "8" 9 | 10 | services: 11 | - docker 12 | 13 | before_install: 14 | - docker pull crate:2.3 15 | - docker run --name crate -d --memory 2g --env CRATE_HEAP_SIZE=1g -p 4200:4200 -p 4300:4300 crate:2.1.5 crate -Ccluster.name=cluster -Clicense.enterprise=false -Cnetwork.host=0.0.0.0 16 | 17 | script: 18 | - sleep 10 19 | - npm test 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | node-crate 2 | ========== 3 | 4 | [![NPM](https://nodei.co/npm/node-crate.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/node-crate/) 5 | 6 | 7 | [![Known Vulnerabilities](https://snyk.io/test/github/megastef/node-crate/badge.svg)](https://snyk.io/test/github/megastef/node-crate) 8 |  [![Standard - JavaScript Style Guide](https://img.shields.io/badge/code%20style-standard-brightgreen.svg)](http://standardjs.com/) 9 |  [![Build Status](https://travis-ci.org/megastef/node-crate.svg?branch=master)](https://travis-ci.org/megastef/node-crate.svg?branch=master) 10 |  [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/megastef/node-crate?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 11 | 12 | This is an independent node.js driver implementation for CRATE using the _sql endpoint REST API. 13 | 14 | [Crate Data](http://crate.io) "Easy to scale real time SQL data store" 15 | 16 | Please note: Crate is a trademark of Crate Technology Gmbh, registered in the E.U. and in other countries. 17 | 18 | 19 | ## Features: 20 | 1. Async Interface 21 | 2. Conversion from rows to array of JSON entities 22 | 3. Automatic build of SQL statements from JSON or full control with SQL String with placeholders and arguments 23 | 4. Support for BLOB objects (e.g. for image uploads) with inbuilt key generation 24 | 25 | ## Known limitation 26 | Nested JSON objects are currently not supported to generate SQL statements (e.g. for insert/update). 27 | We might change this soon. 28 | 29 | ## Breaking changes in version 2 / Migration 30 | 31 | Node-crate now using ES6 features and is not compatible anymore with node.js version lower than 6.0.0. 32 | 33 | Package is using native promises, instead of promises implementation by D.js package. 34 | 35 | Node-crate version 1.x code: 36 | ``` 37 | crate.execute("select ...", {}).success(console.log).error(console.error); 38 | ``` 39 | 40 | Should be updated to node-crate version 2.x: 41 | ``` 42 | crate.execute("select ...", {}).then((res) => {...})).catch((err) => {...})) 43 | ``` 44 | 45 | 46 | ## Installation 47 | 48 | ``` 49 | npm install node-crate 50 | ``` 51 | 52 | ## Test 53 | When a crate instance is running on http://localhost:4200 you can use [lab](https://github.com/spumko/lab) based test (test/test.js). 54 | Test actions: create table, insert, select, update, delete and drop table. 55 | 56 | ``` 57 | npm test 58 | ``` 59 | 60 | ## Usage 61 | 62 | ```js 63 | var crate = require('node-crate'); 64 | crate.connect('localhost', 4200); 65 | // or crate.connect ('http://localhost:4200') 66 | // to use multiple nodes in round robin crate.connect ('http://host1:4200 http://host2:4200') 67 | // to use https crate.connect ('https://host1:4200 https://host2:4200') 68 | crate.execute("select * from tweets where text like ? and retweed=? limit 1", ['Frohe Ostern%', true]).then((res) => { 69 | // res.json is an array with JSON object, with column names as properties, TIMESTAMP is converted to Date for crate V0.38+ 70 | // res.cols are column names 71 | // res.rows values as array of arrays 72 | // res.duration execution time of query 73 | // res.rowcount number of rows 74 | // res.col_types type of column, e.g. res.col_types[i] == crate.type.TIMESTAMP 75 | console.log('Success', res.json, res.duration, res.rowcount, res.cols, res.rows) 76 | }) 77 | 78 | ``` 79 | ### execute (sql, args) 80 | ```js 81 | crate.execute("select * from tweets where text like ?", ['%crate%']).then((res) => console.log(res))).catch((err) => console.log(err)) 82 | ``` 83 | ### insert (tableName, jsonEntity) 84 | ```js 85 | crate.insert('mytable', {columnName1: 'value1', columnName2: 'value2'}).then((res) => {}) 86 | ``` 87 | 88 | ### create (schema) 89 | ```js 90 | var schema = {book: {id: 'integer primary key', title: 'string', author: 'string'}} 91 | crate.create(schema).then(() => {}) 92 | ``` 93 | 94 | ### createIfNotExists (schema) 95 | ```js 96 | var schema = {book: {id: 'integer primary key', title: 'string', author: 'string'}} 97 | crate.createIfNotExists(schema).then(() => {}) 98 | ``` 99 | 100 | ### drop (tableName) 101 | ```js 102 | crate.drop('mytable').then(() => {}) 103 | ``` 104 | 105 | ### update (tableName, jsonEntity, whereClause) 106 | ```js 107 | crate.update('mytable', {columnName1: 'value1', columnName2: 'value2'}, 'columnName3=5').then(() => {}) 108 | ``` 109 | 110 | ### delete (tableName, where) 111 | ```js 112 | crate.delete('mytable', "columnName1='value1'").then(() => {}) 113 | ``` 114 | 115 | ## BLOB's 116 | 117 | ### createBlobTable (tableName, replicas, shards) 118 | ``` 119 | crate.createBlobTable('images',1,3).then((res) => {}).catch((e) => {}) 120 | ``` 121 | 122 | ### insertBlob (tableName, buffer) 123 | ```js 124 | crate.insertBlob('images', buffer).then((res) => {}); 125 | ``` 126 | 127 | ### insertBlobFile (tableName, fileName) 128 | The callback returns the required haskey to get the image with getBlob. 129 | 130 | ```js 131 | crate.insertBlobFile ('images', './test.png').then((hashKey) => { 132 | console.log ("Assigned hashkey": hashKey) 133 | }) 134 | ``` 135 | 136 | ### getBlob (tableName, hashKey) 137 | The callback return a buffer as result - callback (buffer) 138 | ```js 139 | crate.getBlob ('f683e0c9abcbf518704af66c6195bfd3ff121f09').then((data) => { 140 | fs.writeFileSync ('test.gif', data) 141 | }); 142 | ``` 143 | 144 | # Connect to different instances or clusters 145 | 146 | Example connect to localhost and to a crate hosted in a cloud enviroment with authentication 147 | 148 | ```js 149 | 150 | const crateLocal = require('node-crate'); 151 | const crateCloud = crateLocal.getNewInstance(); 152 | 153 | crateLocal.connect('http://localhost:4200'); 154 | crateCloud.connect('https://user:password@cratecloud.com:4200'); 155 | 156 | ``` 157 | 158 | 159 | # Use in Webbrowsers JavaScript 160 | 161 | The intention was to use it with node.js on the server side, but it is possible to make it available in a web browser using [browserify](https://github.com/substack/node-browserify). 162 | ``` 163 | npm run bundle 164 | ``` 165 | 166 | The resulting automatically generated using drone.io. You can refer to this file: 167 | 168 | ``` 169 | 170 | ``` 171 | 172 | Then you might be able to use it inside of an CRATE-Plug-In HTML page: 173 | 174 | ``` 175 | 176 | 180 | ``` 181 | 182 | ## License 183 | 184 | The MIT License(MIT) 185 | Copyright(C) 2014 by Stefan Thies, Igor Likhomanov 186 | 187 | Permission is hereby granted, free of charge, to any person obtaining a copy 188 | of this software and associated documentation files(the "Software"), to deal 189 | in the Software without restriction, including without limitation the rights 190 | to use, copy, modify, merge, publish, distribute, sublicense, and / or sell 191 | copies of the Software, and to permit persons to whom the Software is 192 | furnished to do so, subject to the following conditions: 193 | 194 | The above copyright notice and this permission notice shall be included in 195 | all copies or substantial portions of the Software. 196 | 197 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 198 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 199 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 200 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 201 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 202 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 203 | THE SOFTWARE. 204 | -------------------------------------------------------------------------------- /lib/connection-pool.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | // Currently implemented round robin to the nodes - other startegies, like main node, fallback nodes 4 | // needs to be implemented 5 | const http = require('http') 6 | const https = require('https') 7 | const {URL} = require('url') 8 | /* var optionTemplate = { 9 | host: 'localhost', 10 | path: '/_sql?types', 11 | port: '4200', 12 | method: 'POST', 13 | headers: { 14 | 'Connection': 'keep-alive' 15 | } 16 | } */ 17 | 18 | // limit number of sockets 19 | // http.globalAgent.maxSockets = 3 20 | http.globalAgent.keepAlive = true 21 | 22 | function getRequest (nodeOptions, callback) { 23 | if (nodeOptions.protocol === 'https') { 24 | return https.request(nodeOptions.httpOpt, callback) 25 | } 26 | return http.request(nodeOptions.httpOpt, callback) 27 | } 28 | 29 | function parseStringOptions (opt) { 30 | const nodes = opt.split(' ') 31 | const nodeInfos = nodes.map(node => { 32 | const url = new URL(node); 33 | const urlFields = { 34 | hostname: url.hostname, 35 | port: url.port, 36 | protocol: url.protocol.replace(':', ''), 37 | auth: url.username ? `${url.username}:${url.password}` : null 38 | } 39 | return urlFields 40 | }) 41 | return nodeInfos 42 | } 43 | 44 | class ConnectionPool { 45 | constructor () { 46 | this.httpOptions = [] 47 | this.httpOptionsBlob = [] 48 | this.lastUsed = 0 49 | } 50 | 51 | // optionString e.g. "https://localhost:9200 http://myserver:4200" 52 | connect (optionString) { 53 | const options = parseStringOptions(optionString) 54 | 55 | options.forEach(e => { 56 | this.httpOptions.push({ 57 | httpOpt: { 58 | host: e.hostname, 59 | port: e.port || 4200, 60 | auth: e.auth || null, 61 | path: '/_sql?types', 62 | method: 'POST', 63 | headers: { 64 | Connection: 'keep-alive', 65 | 'Content-Type': 'application/json', 66 | Accept: 'application/json' 67 | } 68 | }, 69 | protocol: e.protocol 70 | }) 71 | 72 | this.httpOptionsBlob.push(`${e.protocol}://${e.hostname}:${e.port}/_blobs/`) 73 | }) 74 | } 75 | 76 | getNextNodeOptions (type) { 77 | this.lastUsed += 1 78 | if (this.lastUsed > this.httpOptions.length - 1) { 79 | this.lastUsed = 0 80 | } 81 | 82 | if (type === 'blob') { 83 | return this.httpOptionsBlob[this.lastUsed] 84 | } 85 | 86 | return this.httpOptions[this.lastUsed] 87 | } 88 | 89 | getSqlRequest (callback) { 90 | const options = this.getNextNodeOptions('sql') 91 | return getRequest(options, callback) 92 | } 93 | 94 | getBlobUrl () { 95 | return this.getNextNodeOptions('blob') 96 | } 97 | 98 | getHttpOptions () { 99 | return this.getNextNodeOptions('sql').httpOpt 100 | } 101 | } 102 | 103 | module.exports = ConnectionPool 104 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | /* 3 | The MIT License(MIT) 4 | Copyright(C) 2014 by Stefan Thies, Igor Likhomanov 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files(the "Software"), 8 | to deal in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and / or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | */ 24 | 25 | const ConnectionPool = require('./connection-pool.js') 26 | const crateTypes = require('./types') 27 | const Type = require('type-of-is') 28 | const http = require('http') 29 | const crypto = require('crypto') 30 | const fs = require('fs') 31 | 32 | const qMarks = '?' 33 | 34 | function getValueByType (v) { 35 | if (Type.is(v, Date)) { 36 | return v.getTime() 37 | } 38 | return v 39 | } 40 | 41 | /** 42 | * @param {string[]} options 43 | * @returns values 44 | * @returns values.keys 45 | * @returns values.values 46 | * @returns values.args 47 | */ 48 | function _prepareOptions (options) { 49 | const values = {} 50 | const keys = Object.keys(options) 51 | values.keys = keys.map(i => '"' + i + '"') 52 | values.values = keys.map(() => qMarks) 53 | values.args = keys.map(i => getValueByType(options[i])) 54 | 55 | return values 56 | } 57 | 58 | class NodeCrate { 59 | constructor () { 60 | this.connectionPool = new ConnectionPool() 61 | this.types = crateTypes 62 | } 63 | 64 | connect (host, port) { 65 | if (port && port >= 0) { 66 | this.connectionPool.connect(`http://${host}:${port}`) 67 | } else { 68 | this.connectionPool.connect(host) 69 | } 70 | } 71 | 72 | /** 73 | * @param {string} tableName 74 | * @param {string[]} options 75 | */ 76 | insert (tableName, options) { 77 | if (arguments.length < 2) { 78 | return Promise.reject(new Error('missed arguments!')) 79 | } 80 | 81 | if (!tableName) { 82 | return Promise.reject(new Error('Table name is not specified')) 83 | } 84 | 85 | if (!options) { 86 | return Promise.reject(new Error('Record entry is not defined')) 87 | } 88 | 89 | const preparedOptions = _prepareOptions(options) 90 | const preparedQuery = `INSERT INTO ${tableName} (${preparedOptions.keys}) VALUES (${preparedOptions.values})` 91 | 92 | return this._executeSql(preparedQuery, preparedOptions.args) 93 | } 94 | 95 | /** 96 | * @param {string} tableName 97 | * @param {string[]} options 98 | * @param {string} whereClause 99 | */ 100 | update (tableName, options, whereClause) { 101 | if (arguments.length < 3) { 102 | return Promise.reject(new Error('missed arguments!')) 103 | } 104 | 105 | if (!tableName) { 106 | return Promise.reject(new Error('Table name is not specified!')) 107 | } 108 | 109 | if (!options) { 110 | return Promise.reject(new Error('Record entry is not defined')) 111 | } 112 | 113 | if (!whereClause) { 114 | return Promise.reject(new Error('Where clause is not defined')) 115 | } 116 | 117 | const preparedOptions = _prepareOptions(options) 118 | const setStmtParts = [] 119 | 120 | for (let i = 0; i < preparedOptions.keys.length; ++i) { 121 | setStmtParts.push( 122 | `${preparedOptions.keys[i]}=${preparedOptions.values[i]}` 123 | ) 124 | } 125 | 126 | const preparedQuery = `UPDATE ${tableName} SET ${setStmtParts.join( 127 | ' , ' 128 | )} WHERE ${whereClause}` 129 | 130 | return this._executeSql(preparedQuery, preparedOptions.args) 131 | } 132 | 133 | /** 134 | * @param {string} tableName 135 | * @param {string} whereClause 136 | */ 137 | delete (tableName, whereClause) { 138 | if (arguments.length < 2) { 139 | return Promise.reject(new Error('missed arguments!')) 140 | } 141 | 142 | if (!tableName) { 143 | return Promise.reject(new Error('Table name is not specified!')) 144 | } 145 | 146 | if (!whereClause) { 147 | return Promise.reject(new Error('Where clause is not defined')) 148 | } 149 | 150 | const preparedQuery = `DELETE FROM ${tableName} WHERE ${whereClause}` 151 | 152 | return this._executeSql(preparedQuery) 153 | } 154 | 155 | /** 156 | * @param {string} tableName 157 | */ 158 | drop (tableName) { 159 | if (!tableName) { 160 | return Promise.reject(new Error('Table name is not specified!')) 161 | } 162 | 163 | const preparedQuery = `DROP TABLE ${tableName}` 164 | return this._executeSql(preparedQuery) 165 | } 166 | 167 | /** 168 | * @param {string} tableName 169 | */ 170 | dropBlobTable (tableName) { 171 | if (!tableName) { 172 | return Promise.reject(new Error('Table name is not specified!')) 173 | } 174 | 175 | const preparedQuery = `DROP BLOB TABLE ${tableName}` 176 | return this._executeSql(preparedQuery) 177 | } 178 | 179 | /** 180 | * @param {string} sql statement 181 | * @param {array} args (optional) 182 | */ 183 | execute (sql, args = []) { 184 | return this._executeSql(sql, args) 185 | } 186 | 187 | executeBulk (sql, bulkArgs = []) { 188 | return this._executeSql(sql, bulkArgs, true) 189 | } 190 | 191 | /** 192 | * @param {string} tableName 193 | * @param {string} buffer 194 | */ 195 | insertBlob (tableName, buffer) { 196 | return new Promise((resolve, reject) => { 197 | const hashCode = crypto 198 | .createHash('sha1') 199 | .update(buffer, 'binary') 200 | .digest('hex') 201 | 202 | const callback = function (response) { 203 | let body = '' 204 | 205 | response.on('data', data => { 206 | body += data.toString() 207 | return undefined 208 | }) 209 | 210 | response.on('error', err => { 211 | return reject(err) 212 | }) 213 | 214 | response.on('end', () => { 215 | // if the object already exists CRATE returns 409 status code 216 | if (response.statusCode === 409) { 217 | return reject(new Error('error 409: already exists')) 218 | } 219 | 220 | // if everything is alreight CRATE returns '201 created' status 221 | if (response.statusCode > 299) { 222 | return reject( 223 | new Error( 224 | `error HTTP status code: ${response.statusCode} ${body}` 225 | ) 226 | ) 227 | } 228 | 229 | return resolve(hashCode) 230 | }) 231 | } 232 | 233 | const req = this.createBlobWriteStream(tableName, hashCode, callback) 234 | 235 | try { 236 | req.write(buffer) 237 | req.end() 238 | } catch (ex) { 239 | return reject(ex) 240 | } 241 | }) 242 | } 243 | 244 | /** 245 | * @param {string} tableName 246 | * @param {string} filename 247 | */ 248 | insertBlobFile (tableName, filename) { 249 | return new Promise((resolve, reject) => { 250 | fs.readFile(filename, (err, data) => { 251 | if (err) { 252 | return reject(err) 253 | } 254 | 255 | this.insertBlob(tableName, data) 256 | .then(res => resolve(res)) 257 | .catch(err => reject(err)) 258 | }) 259 | }) 260 | } 261 | 262 | /** 263 | * @param {string} tableName 264 | * @param {string} hashKey 265 | */ 266 | getBlob (tableName, hashKey) { 267 | return new Promise((resolve, reject) => { 268 | const callback = function (response) { 269 | const buffer = [] 270 | 271 | response.on('data', chunk => { 272 | buffer.push(chunk) 273 | }) 274 | 275 | response.on('error', err => { 276 | return reject(err) 277 | }) 278 | 279 | response.on('end', () => { 280 | return resolve(Buffer.concat(buffer)) 281 | }) 282 | } 283 | 284 | const reqUrl = `${this.connectionPool.getBlobUrl()}${tableName}/${hashKey}` 285 | http.get(reqUrl, callback) 286 | }) 287 | } 288 | 289 | /** 290 | * @param {object} schema like: {person: {name: 'string', age: 'integer'}} 291 | */ 292 | create (schema) { 293 | const tableName = Object.keys(schema)[0] 294 | const table = schema[tableName] 295 | const cols = Object.keys(table).map(key => { 296 | return `"${key}" ${table[key]}` 297 | }) 298 | 299 | const statement = `CREATE TABLE ${tableName} (${cols})` 300 | return this._executeSql(statement) 301 | } 302 | 303 | /** 304 | * @param {object} schema like: {person: {name: 'string', age: 'integer'}} 305 | */ 306 | createIfNotExists (schema) { 307 | const tableName = Object.keys(schema)[0] 308 | const table = schema[tableName] 309 | const cols = Object.keys(table).map(key => { 310 | return `"${key}" ${table[key]}` 311 | }) 312 | 313 | const statement = `CREATE TABLE IF NOT EXISTS ${tableName} (${cols})` 314 | return this._executeSql(statement) 315 | } 316 | 317 | /** 318 | * @param {tableName} Name of the BLOB Table 319 | * @param {replicas} Number of replicas 320 | * @param {shards} Number of shards 321 | */ 322 | createBlobTable (tableName, replicas, shards) { 323 | const statement = `CREATE BLOB TABLE ${tableName} clustered into ${shards} shards with (number_of_replicas=${replicas})` 324 | return this._executeSql(statement) 325 | } 326 | 327 | /** 328 | * @param {string} sql 329 | * @param {string[]} args 330 | */ 331 | _executeSql (sql, args = [], bulk = false) { 332 | return new Promise((resolve, reject) => { 333 | const callback = function (response) { 334 | const data = [] 335 | 336 | response.on('data', chunk => { 337 | data.push(chunk) 338 | }) 339 | 340 | response.on('error', err => { 341 | return reject(new Error(err)) 342 | }) 343 | 344 | response.on('end', () => { 345 | let result = {} 346 | let json 347 | 348 | try { 349 | result = JSON.parse(data.join('')) 350 | } catch (ex) { 351 | return reject(new Error(ex)) 352 | } 353 | 354 | if (!result.rows) { 355 | /* || /CREATE BLOB/im.test (sql)) */ 356 | // workaround CRATE does not return a row when it creates a BLOB 357 | result.rows = [] 358 | } 359 | 360 | if (result.error) { 361 | return reject(result.error) 362 | } 363 | 364 | /* jslint nomen: true */ 365 | result.__defineGetter__('json', () => { 366 | if (!json) { 367 | json = result.rows.map(e => { 368 | const x = {} 369 | for (let i = 0; i < result.cols.length; ++i) { 370 | if ( 371 | result.col_types && 372 | (result.col_types[i] === crateTypes.TIMESTAMP || 373 | result.col_types[i] === crateTypes.TIMESTAMPTZ) 374 | ) { 375 | x[result.cols[i]] = new Date(e[i]) 376 | } else { 377 | x[result.cols[i]] = e[i] 378 | } 379 | } 380 | 381 | return x 382 | }) 383 | } 384 | 385 | return json 386 | }) 387 | 388 | /* jslint nomen: false */ 389 | return resolve(result) 390 | }) 391 | } 392 | 393 | const req = this.connectionPool.getSqlRequest(callback) 394 | // let node know what to do in case of unexpected errors 395 | // before anything has been sent/received. 396 | req.on('error', err => { 397 | console.log('Unexpected connection error', err) 398 | req.end() 399 | return reject(err) 400 | }) 401 | let command = { 402 | stmt: sql 403 | } 404 | 405 | if (bulk === true) { 406 | command.bulk_args = args 407 | } else { 408 | command.args = args 409 | } 410 | 411 | command = JSON.stringify(command) 412 | 413 | try { 414 | req.write(command) 415 | req.end() 416 | } catch (ex) { 417 | return reject(ex) 418 | } 419 | }) 420 | } 421 | 422 | /** 423 | * @param {string} tableName 424 | * @param {string} streamId - unique ID for the blob 425 | * @param {requestCallback} cb - callback function gor the http request 426 | * @returns http.request object (writeable) 427 | */ 428 | createBlobWriteStream (tableName, streamId, cb) { 429 | const options = this.connectionPool.getHttpOptions() 430 | const blobOptions = { 431 | host: options.host, 432 | path: `/_blobs/${tableName}/${streamId}`, 433 | port: options.port, 434 | method: 'PUT', 435 | headers: { 'Conent-Type': 'application/binary' } 436 | } 437 | 438 | return http.request(blobOptions, cb) 439 | } 440 | 441 | getNewInstance () { 442 | return new NodeCrate() 443 | } 444 | } 445 | 446 | module.exports = new NodeCrate() 447 | -------------------------------------------------------------------------------- /lib/types.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = { 4 | // NULL: 0, 5 | UNDEFINED: 0, 6 | NOT_SUPPORTED: 1, 7 | CHAR: 2, 8 | BOOLEAN: 3, 9 | STRING: 4, 10 | IP: 5, 11 | DOUBLE: 6, 12 | FLOAT: 7, 13 | SHORT: 8, 14 | INTEGER: 9, 15 | LONG: 10, 16 | TIMESTAMPTZ: 11, 17 | OBJECT: 12, 18 | GEO_POINT: 13, 19 | GEO_SHAPE: 14, 20 | TIMESTAMP: 15, 21 | REGPROC: 19, 22 | TIME: 20, 23 | OIDVECTOR: 21, 24 | NUMERIC: 22, 25 | REGCLASS: 23, 26 | DATE: 24, 27 | BIT: 25, 28 | JSON: 26, 29 | CHARACTER: 27, 30 | ARRAY: 100, 31 | SET: 101 32 | } 33 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "node-crate", 3 | "version": "3.0.0", 4 | "description": "Node.js SQL-driver for CrateDB (crate.io)", 5 | "main": "lib/index.js", 6 | "scripts": { 7 | "test": "standard --format lib/*.js && mocha -t 20000", 8 | "bundle": "browserify -r ./lib/index.js:node-crate > bundle.js", 9 | "fix": "standard --fix lib/*.js" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "https://github.com/megastef/node-crate.git" 14 | }, 15 | "keywords": [ 16 | "CRATE", 17 | "CrateDB", 18 | "crate.io", 19 | "sql", 20 | "driver", 21 | "bigdata", 22 | "elasticsearch alternative", 23 | "driver for scalable data store", 24 | "BLOB storage" 25 | ], 26 | "author": "Stefan Thies", 27 | "contributors": [ 28 | "Igor Likhomanov", 29 | "Martin Heidegger" 30 | ], 31 | "license": "MIT License", 32 | "bugs": { 33 | "url": "https://github.com/megastef/node-crate/issues" 34 | }, 35 | "homepage": "http://megastef.github.io/node-crate/", 36 | "dependencies": { 37 | "type-of-is": ">= 3.3.x" 38 | }, 39 | "devDependencies": { 40 | "expect": "^29.5.0", 41 | "mocha": "^10.2.0", 42 | "standard": "^17.0.0" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | /* global describe, it */ 4 | const {expect} = require('expect') 5 | 6 | const crate = require('../') 7 | 8 | // Why only 50? the default setting in crate ... 9 | // EsThreadPoolExecutor[bulk, queue capacity = 50] 10 | // for more than 50 inserts at once use Bulk insert or increase queue in Crate 11 | const docsToInsert = 50 12 | 13 | crate.connect(process.env.CRATE_URL || 'http://127.0.0.1:4200') 14 | 15 | const blobTableName = 'blob_test_3' 16 | const tableName = 'NodeCrateTest_3' 17 | 18 | describe('#node-crate', function () { 19 | this.slow(15000) 20 | 21 | it('should create blob table', async () => { 22 | const res = await crate.createBlobTable(blobTableName, 0, 1) 23 | expect(res.rowcount).toBe(1) 24 | }); 25 | 26 | it('should create table', async () => { 27 | const schema = {} 28 | schema[tableName] = { id: 'integer primary key', title: 'string', numberVal: 'integer' } 29 | 30 | const res = await crate.create(schema) 31 | expect(res.rowcount).toBe(1) 32 | }); 33 | 34 | it('should create table if not exists - table exists', async () => { 35 | const schema = {} 36 | schema[tableName] = { id: 'integer primary key', title: 'string', numberVal: 'integer' } 37 | 38 | const res = await crate.createIfNotExists(schema) 39 | expect(res.rowcount).toBe(0) 40 | }) 41 | 42 | it('should drop table to create again', async () => { 43 | const res = await crate.drop(tableName) 44 | expect(res.rowcount).toBe(1) 45 | }) 46 | 47 | it('should create table if not exists - table does not exist', async () => { 48 | const schema = {} 49 | schema[tableName] = { id: 'integer primary key', title: 'string', numberVal: 'integer' } 50 | 51 | const res = await crate.createIfNotExists(schema) 52 | expect(res.rowcount).toBe(1) 53 | }) 54 | 55 | let hashkey = '' 56 | 57 | it('should insert blob', async () => { 58 | const res = await crate.insertBlobFile(blobTableName, './lib/index.js'); 59 | expect(res.length).toBe(40); 60 | hashkey = res; 61 | }); 62 | 63 | it('should insert', async () => { 64 | const res = await crate.insert(tableName, { 65 | id: '1', 66 | title: 'Title', 67 | numberVal: 42 68 | }); 69 | expect(res.rowcount).toBe(1); 70 | }); 71 | 72 | it('should insert many', async () => { 73 | let success = 0 74 | let errorReported = false 75 | let longTitle = 'A long title to generate larger chunks ...' 76 | 77 | for (let k = 0; k < 5; ++k) { 78 | longTitle += longTitle 79 | } 80 | 81 | for (let i = 0; i < docsToInsert; ++i) { 82 | await crate.insert(tableName, { 83 | id: i + 100, 84 | title: longTitle, 85 | numberVal: 42 86 | }) 87 | .then(() => { 88 | success++; 89 | }) 90 | .catch((err) => { 91 | throw err; 92 | }); 93 | } 94 | expect(success).toBe(docsToInsert); 95 | }); 96 | 97 | it('should insert bulk documents', async () => { 98 | const title = 'A title'; 99 | const bulkArgs = []; 100 | 101 | for (let i = 0; i < docsToInsert; ++i) { 102 | bulkArgs[i] = [i + 1000, title, 42]; 103 | } 104 | 105 | const res = await crate.executeBulk(`INSERT INTO ${tableName} ("id","title","numberVal") Values (?, ?, ?)`, bulkArgs); 106 | expect(res.results.length).toBe(docsToInsert); 107 | }); 108 | 109 | it('should select', async () => { 110 | await crate.execute(`REFRESH TABLE ${tableName}`); 111 | const res = await crate.execute(`SELECT * FROM ${tableName} limit ${docsToInsert}`); 112 | expect(res.rowcount).toBe(docsToInsert); 113 | }); 114 | 115 | it('should update', async () => { 116 | const res = await crate.update(tableName, { 117 | title: 'TitleNew' 118 | }, 'id=1'); 119 | expect(res.rowcount).toBe(1); 120 | }); 121 | 122 | it('should select after update', async () => { 123 | await crate.execute(`REFRESH TABLE ${tableName}`); 124 | const res = await crate.execute(`SELECT * FROM ${tableName} where id=1 limit 100`); 125 | expect(res.json[0].title).toBe('TitleNew'); 126 | expect(res.json[0].numberVal).toBe(42); 127 | }); 128 | 129 | it('should get blob', async () => { 130 | const res = await crate.getBlob('blobtest', hashkey); 131 | expect(res instanceof Buffer).toBe(true); 132 | }); 133 | 134 | it('should delete', async () => { 135 | const res = await crate.delete(tableName, 'id=1'); 136 | await crate.execute(`REFRESH TABLE ${tableName}`); 137 | expect(res.rowcount).toBe(1); 138 | }); 139 | 140 | it('should drop table', async () => { 141 | const res = await crate.drop(tableName); 142 | expect(res.rowcount).toBe(1); 143 | }); 144 | 145 | it('should drop blob table', async () => { 146 | const res = await crate.dropBlobTable(blobTableName); 147 | expect(res.rowcount).toBe(1); 148 | }); 149 | 150 | it('should fail to drop blob table with empty name', async () => { 151 | try { 152 | await crate.dropBlobTable(); 153 | } catch (err) { 154 | expect(err.message).toBe('Table name is not specified!'); 155 | } 156 | }); 157 | }) 158 | --------------------------------------------------------------------------------