├── .gitignore
├── .jshintrc
├── .travis.yml
├── README.md
├── lib
├── connection-pool.js
├── index.js
└── types.js
├── package-lock.json
├── package.json
└── test
└── test.js
/.gitignore:
--------------------------------------------------------------------------------
1 | .idea/
2 | node_modules/
3 | test/index.html
4 | bundle.js
5 | .gittoken
6 |
--------------------------------------------------------------------------------
/.jshintrc:
--------------------------------------------------------------------------------
1 | {
2 | // JSHint Default Configuration File (as on JSHint website)
3 | // See http://jshint.com/docs/ for more details
4 |
5 | "maxerr" : 50, // {int} Maximum error before stopping
6 | "esversion" : 6,
7 |
8 | // Enforcing
9 | "bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.)
10 | "camelcase" : false, // true: Identifiers must be in camelCase
11 | "curly" : true, // true: Require {} for every new block or scope
12 | "eqeqeq" : true, // true: Require triple equals (===) for comparison
13 | "forin" : false, // true: Require filtering for..in loops with obj.hasOwnProperty()
14 | "freeze" : true, // true: prohibits overwriting prototypes of native objects such as Array, Date etc.
15 | "immed" : false, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());`
16 | "indent" : 4, // {int} Number of spaces to use for indentation
17 | "latedef" : false, // true: Require variables/functions to be defined before being used
18 | "newcap" : false, // true: Require capitalization of all constructor functions e.g. `new F()`
19 | "noarg" : true, // true: Prohibit use of `arguments.caller` and `arguments.callee`
20 | "noempty" : true, // true: Prohibit use of empty blocks
21 | "nonbsp" : true, // true: Prohibit "non-breaking whitespace" characters.
22 | "nonew" : false, // true: Prohibit use of constructors for side-effects (without assignment)
23 | "plusplus" : false, // true: Prohibit use of `++` & `--`
24 | "quotmark" : false, // Quotation mark consistency:
25 | // false : do nothing (default)
26 | // true : ensure whatever is used is consistent
27 | // "single" : require single quotes
28 | // "double" : require double quotes
29 | "undef" : true, // true: Require all non-global variables to be declared (prevents global leaks)
30 | "unused" : true, // true: Require all defined variables be used
31 | "strict" : false, // true: Requires all functions run in ES5 Strict Mode
32 | "maxparams" : false, // {int} Max number of formal params allowed per function
33 | "maxdepth" : false, // {int} Max depth of nested blocks (within functions)
34 | "maxstatements" : false, // {int} Max number statements per function
35 | "maxcomplexity" : false, // {int} Max cyclomatic complexity per function
36 | "maxlen" : false, // {int} Max number of characters per line
37 |
38 | // Relaxing
39 | "asi" : true, // true: Tolerate Automatic Semicolon Insertion (no semicolons)
40 | "boss" : false, // true: Tolerate assignments where comparisons would be expected
41 | "debug" : false, // true: Allow debugger statements e.g. browser breakpoints.
42 | "eqnull" : false, // true: Tolerate use of `== null`
43 | "es5" : false, // true: Allow ES5 syntax (ex: getters and setters)
44 | "esnext" : false, // true: Allow ES.next (ES6) syntax (ex: `const`)
45 | "moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features)
46 | // (ex: `for each`, multiple try/catch, function expression…)
47 | "evil" : false, // true: Tolerate use of `eval` and `new Function()`
48 | "expr" : false, // true: Tolerate `ExpressionStatement` as Programs
49 | "funcscope" : false, // true: Tolerate defining variables inside control statements
50 | "globalstrict" : false, // true: Allow global "use strict" (also enables 'strict')
51 | "iterator" : false, // true: Tolerate using the `__iterator__` property
52 | "lastsemic" : false, // true: Tolerate omitting a semicolon for the last statement of a 1-line block
53 | "laxbreak" : false, // true: Tolerate possibly unsafe line breakings
54 | "laxcomma" : false, // true: Tolerate comma-first style coding
55 | "loopfunc" : false, // true: Tolerate functions being defined in loops
56 | "multistr" : false, // true: Tolerate multi-line strings
57 | "noyield" : false, // true: Tolerate generator functions with no yield statement in them.
58 | "notypeof" : false, // true: Tolerate invalid typeof operator values
59 | "proto" : false, // true: Tolerate using the `__proto__` property
60 | "scripturl" : false, // true: Tolerate script-targeted URLs
61 | "shadow" : false, // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
62 | "sub" : true, // true: Tolerate using `[]` notation when it can still be expressed in dot notation
63 | "supernew" : false, // true: Tolerate `new function () { ... };` and `new Object;`
64 | "validthis" : false, // true: Tolerate using this in a non-constructor function
65 |
66 | // Environments
67 | "browser" : true, // Web Browser (window, document, etc)
68 | "browserify" : false, // Browserify (node.js code in the browser)
69 | "couch" : false, // CouchDB
70 | "devel" : true, // Development/debugging (alert, confirm, etc)
71 | "dojo" : false, // Dojo Toolkit
72 | "jasmine" : false, // Jasmine
73 | "jquery" : false, // jQuery
74 | "mocha" : true, // Mocha
75 | "mootools" : false, // MooTools
76 | "node" : true, // Node.js
77 | "nonstandard" : false, // Widely adopted globals (escape, unescape, etc)
78 | "prototypejs" : false, // Prototype and Scriptaculous
79 | "qunit" : false, // QUnit
80 | "rhino" : false, // Rhino
81 | "shelljs" : false, // ShellJS
82 | "worker" : false, // Web Workers
83 | "wsh" : false, // Windows Scripting Host
84 | "yui" : false, // Yahoo User Interface
85 |
86 | // Custom Globals
87 | "globals" : {} // additional predefined global variables
88 | }
89 |
90 |
91 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | sudo: required
2 |
3 | language: node_js
4 | node_js:
5 | - "node"
6 | - "6"
7 | - "7"
8 | - "8"
9 |
10 | services:
11 | - docker
12 |
13 | before_install:
14 | - docker pull crate:2.3
15 | - docker run --name crate -d --memory 2g --env CRATE_HEAP_SIZE=1g -p 4200:4200 -p 4300:4300 crate:2.1.5 crate -Ccluster.name=cluster -Clicense.enterprise=false -Cnetwork.host=0.0.0.0
16 |
17 | script:
18 | - sleep 10
19 | - npm test
20 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | node-crate
2 | ==========
3 |
4 | [](https://nodei.co/npm/node-crate/)
5 |
6 |
7 | [](https://snyk.io/test/github/megastef/node-crate)
8 | [](http://standardjs.com/)
9 | [](https://travis-ci.org/megastef/node-crate.svg?branch=master)
10 | [](https://gitter.im/megastef/node-crate?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
11 |
12 | This is an independent node.js driver implementation for CRATE using the _sql endpoint REST API.
13 |
14 | [Crate Data](http://crate.io) "Easy to scale real time SQL data store"
15 |
16 | Please note: Crate is a trademark of Crate Technology Gmbh, registered in the E.U. and in other countries.
17 |
18 |
19 | ## Features:
20 | 1. Async Interface
21 | 2. Conversion from rows to array of JSON entities
22 | 3. Automatic build of SQL statements from JSON or full control with SQL String with placeholders and arguments
23 | 4. Support for BLOB objects (e.g. for image uploads) with inbuilt key generation
24 |
25 | ## Known limitation
26 | Nested JSON objects are currently not supported to generate SQL statements (e.g. for insert/update).
27 | We might change this soon.
28 |
29 | ## Breaking changes in version 2 / Migration
30 |
31 | Node-crate now using ES6 features and is not compatible anymore with node.js version lower than 6.0.0.
32 |
33 | Package is using native promises, instead of promises implementation by D.js package.
34 |
35 | Node-crate version 1.x code:
36 | ```
37 | crate.execute("select ...", {}).success(console.log).error(console.error);
38 | ```
39 |
40 | Should be updated to node-crate version 2.x:
41 | ```
42 | crate.execute("select ...", {}).then((res) => {...})).catch((err) => {...}))
43 | ```
44 |
45 |
46 | ## Installation
47 |
48 | ```
49 | npm install node-crate
50 | ```
51 |
52 | ## Test
53 | When a crate instance is running on http://localhost:4200 you can use [lab](https://github.com/spumko/lab) based test (test/test.js).
54 | Test actions: create table, insert, select, update, delete and drop table.
55 |
56 | ```
57 | npm test
58 | ```
59 |
60 | ## Usage
61 |
62 | ```js
63 | var crate = require('node-crate');
64 | crate.connect('localhost', 4200);
65 | // or crate.connect ('http://localhost:4200')
66 | // to use multiple nodes in round robin crate.connect ('http://host1:4200 http://host2:4200')
67 | // to use https crate.connect ('https://host1:4200 https://host2:4200')
68 | crate.execute("select * from tweets where text like ? and retweed=? limit 1", ['Frohe Ostern%', true]).then((res) => {
69 | // res.json is an array with JSON object, with column names as properties, TIMESTAMP is converted to Date for crate V0.38+
70 | // res.cols are column names
71 | // res.rows values as array of arrays
72 | // res.duration execution time of query
73 | // res.rowcount number of rows
74 | // res.col_types type of column, e.g. res.col_types[i] == crate.type.TIMESTAMP
75 | console.log('Success', res.json, res.duration, res.rowcount, res.cols, res.rows)
76 | })
77 |
78 | ```
79 | ### execute (sql, args)
80 | ```js
81 | crate.execute("select * from tweets where text like ?", ['%crate%']).then((res) => console.log(res))).catch((err) => console.log(err))
82 | ```
83 | ### insert (tableName, jsonEntity)
84 | ```js
85 | crate.insert('mytable', {columnName1: 'value1', columnName2: 'value2'}).then((res) => {})
86 | ```
87 |
88 | ### create (schema)
89 | ```js
90 | var schema = {book: {id: 'integer primary key', title: 'string', author: 'string'}}
91 | crate.create(schema).then(() => {})
92 | ```
93 |
94 | ### createIfNotExists (schema)
95 | ```js
96 | var schema = {book: {id: 'integer primary key', title: 'string', author: 'string'}}
97 | crate.createIfNotExists(schema).then(() => {})
98 | ```
99 |
100 | ### drop (tableName)
101 | ```js
102 | crate.drop('mytable').then(() => {})
103 | ```
104 |
105 | ### update (tableName, jsonEntity, whereClause)
106 | ```js
107 | crate.update('mytable', {columnName1: 'value1', columnName2: 'value2'}, 'columnName3=5').then(() => {})
108 | ```
109 |
110 | ### delete (tableName, where)
111 | ```js
112 | crate.delete('mytable', "columnName1='value1'").then(() => {})
113 | ```
114 |
115 | ## BLOB's
116 |
117 | ### createBlobTable (tableName, replicas, shards)
118 | ```
119 | crate.createBlobTable('images',1,3).then((res) => {}).catch((e) => {})
120 | ```
121 |
122 | ### insertBlob (tableName, buffer)
123 | ```js
124 | crate.insertBlob('images', buffer).then((res) => {});
125 | ```
126 |
127 | ### insertBlobFile (tableName, fileName)
128 | The callback returns the required haskey to get the image with getBlob.
129 |
130 | ```js
131 | crate.insertBlobFile ('images', './test.png').then((hashKey) => {
132 | console.log ("Assigned hashkey": hashKey)
133 | })
134 | ```
135 |
136 | ### getBlob (tableName, hashKey)
137 | The callback return a buffer as result - callback (buffer)
138 | ```js
139 | crate.getBlob ('f683e0c9abcbf518704af66c6195bfd3ff121f09').then((data) => {
140 | fs.writeFileSync ('test.gif', data)
141 | });
142 | ```
143 |
144 | # Connect to different instances or clusters
145 |
146 | Example connect to localhost and to a crate hosted in a cloud enviroment with authentication
147 |
148 | ```js
149 |
150 | const crateLocal = require('node-crate');
151 | const crateCloud = crateLocal.getNewInstance();
152 |
153 | crateLocal.connect('http://localhost:4200');
154 | crateCloud.connect('https://user:password@cratecloud.com:4200');
155 |
156 | ```
157 |
158 |
159 | # Use in Webbrowsers JavaScript
160 |
161 | The intention was to use it with node.js on the server side, but it is possible to make it available in a web browser using [browserify](https://github.com/substack/node-browserify).
162 | ```
163 | npm run bundle
164 | ```
165 |
166 | The resulting automatically generated using drone.io. You can refer to this file:
167 |
168 | ```
169 |
170 | ```
171 |
172 | Then you might be able to use it inside of an CRATE-Plug-In HTML page:
173 |
174 | ```
175 |
176 |
180 | ```
181 |
182 | ## License
183 |
184 | The MIT License(MIT)
185 | Copyright(C) 2014 by Stefan Thies, Igor Likhomanov
186 |
187 | Permission is hereby granted, free of charge, to any person obtaining a copy
188 | of this software and associated documentation files(the "Software"), to deal
189 | in the Software without restriction, including without limitation the rights
190 | to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
191 | copies of the Software, and to permit persons to whom the Software is
192 | furnished to do so, subject to the following conditions:
193 |
194 | The above copyright notice and this permission notice shall be included in
195 | all copies or substantial portions of the Software.
196 |
197 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
198 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
199 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
200 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
201 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
202 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
203 | THE SOFTWARE.
204 |
--------------------------------------------------------------------------------
/lib/connection-pool.js:
--------------------------------------------------------------------------------
1 | 'use strict'
2 |
3 | // Currently implemented round robin to the nodes - other startegies, like main node, fallback nodes
4 | // needs to be implemented
5 | const http = require('http')
6 | const https = require('https')
7 | const {URL} = require('url')
8 | /* var optionTemplate = {
9 | host: 'localhost',
10 | path: '/_sql?types',
11 | port: '4200',
12 | method: 'POST',
13 | headers: {
14 | 'Connection': 'keep-alive'
15 | }
16 | } */
17 |
18 | // limit number of sockets
19 | // http.globalAgent.maxSockets = 3
20 | http.globalAgent.keepAlive = true
21 |
22 | function getRequest (nodeOptions, callback) {
23 | if (nodeOptions.protocol === 'https') {
24 | return https.request(nodeOptions.httpOpt, callback)
25 | }
26 | return http.request(nodeOptions.httpOpt, callback)
27 | }
28 |
29 | function parseStringOptions (opt) {
30 | const nodes = opt.split(' ')
31 | const nodeInfos = nodes.map(node => {
32 | const url = new URL(node);
33 | const urlFields = {
34 | hostname: url.hostname,
35 | port: url.port,
36 | protocol: url.protocol.replace(':', ''),
37 | auth: url.username ? `${url.username}:${url.password}` : null
38 | }
39 | return urlFields
40 | })
41 | return nodeInfos
42 | }
43 |
44 | class ConnectionPool {
45 | constructor () {
46 | this.httpOptions = []
47 | this.httpOptionsBlob = []
48 | this.lastUsed = 0
49 | }
50 |
51 | // optionString e.g. "https://localhost:9200 http://myserver:4200"
52 | connect (optionString) {
53 | const options = parseStringOptions(optionString)
54 |
55 | options.forEach(e => {
56 | this.httpOptions.push({
57 | httpOpt: {
58 | host: e.hostname,
59 | port: e.port || 4200,
60 | auth: e.auth || null,
61 | path: '/_sql?types',
62 | method: 'POST',
63 | headers: {
64 | Connection: 'keep-alive',
65 | 'Content-Type': 'application/json',
66 | Accept: 'application/json'
67 | }
68 | },
69 | protocol: e.protocol
70 | })
71 |
72 | this.httpOptionsBlob.push(`${e.protocol}://${e.hostname}:${e.port}/_blobs/`)
73 | })
74 | }
75 |
76 | getNextNodeOptions (type) {
77 | this.lastUsed += 1
78 | if (this.lastUsed > this.httpOptions.length - 1) {
79 | this.lastUsed = 0
80 | }
81 |
82 | if (type === 'blob') {
83 | return this.httpOptionsBlob[this.lastUsed]
84 | }
85 |
86 | return this.httpOptions[this.lastUsed]
87 | }
88 |
89 | getSqlRequest (callback) {
90 | const options = this.getNextNodeOptions('sql')
91 | return getRequest(options, callback)
92 | }
93 |
94 | getBlobUrl () {
95 | return this.getNextNodeOptions('blob')
96 | }
97 |
98 | getHttpOptions () {
99 | return this.getNextNodeOptions('sql').httpOpt
100 | }
101 | }
102 |
103 | module.exports = ConnectionPool
104 |
--------------------------------------------------------------------------------
/lib/index.js:
--------------------------------------------------------------------------------
1 | 'use strict'
2 | /*
3 | The MIT License(MIT)
4 | Copyright(C) 2014 by Stefan Thies, Igor Likhomanov
5 |
6 | Permission is hereby granted, free of charge, to any person obtaining a copy
7 | of this software and associated documentation files(the "Software"),
8 | to deal in the Software without restriction, including without limitation the rights
9 | to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
10 | copies of the Software, and to permit persons to whom the Software is
11 | furnished to do so, subject to the following conditions:
12 |
13 | The above copyright notice and this permission notice shall be included in
14 | all copies or substantial portions of the Software.
15 |
16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 | THE SOFTWARE.
23 | */
24 |
25 | const ConnectionPool = require('./connection-pool.js')
26 | const crateTypes = require('./types')
27 | const Type = require('type-of-is')
28 | const http = require('http')
29 | const crypto = require('crypto')
30 | const fs = require('fs')
31 |
32 | const qMarks = '?'
33 |
34 | function getValueByType (v) {
35 | if (Type.is(v, Date)) {
36 | return v.getTime()
37 | }
38 | return v
39 | }
40 |
41 | /**
42 | * @param {string[]} options
43 | * @returns values
44 | * @returns values.keys
45 | * @returns values.values
46 | * @returns values.args
47 | */
48 | function _prepareOptions (options) {
49 | const values = {}
50 | const keys = Object.keys(options)
51 | values.keys = keys.map(i => '"' + i + '"')
52 | values.values = keys.map(() => qMarks)
53 | values.args = keys.map(i => getValueByType(options[i]))
54 |
55 | return values
56 | }
57 |
58 | class NodeCrate {
59 | constructor () {
60 | this.connectionPool = new ConnectionPool()
61 | this.types = crateTypes
62 | }
63 |
64 | connect (host, port) {
65 | if (port && port >= 0) {
66 | this.connectionPool.connect(`http://${host}:${port}`)
67 | } else {
68 | this.connectionPool.connect(host)
69 | }
70 | }
71 |
72 | /**
73 | * @param {string} tableName
74 | * @param {string[]} options
75 | */
76 | insert (tableName, options) {
77 | if (arguments.length < 2) {
78 | return Promise.reject(new Error('missed arguments!'))
79 | }
80 |
81 | if (!tableName) {
82 | return Promise.reject(new Error('Table name is not specified'))
83 | }
84 |
85 | if (!options) {
86 | return Promise.reject(new Error('Record entry is not defined'))
87 | }
88 |
89 | const preparedOptions = _prepareOptions(options)
90 | const preparedQuery = `INSERT INTO ${tableName} (${preparedOptions.keys}) VALUES (${preparedOptions.values})`
91 |
92 | return this._executeSql(preparedQuery, preparedOptions.args)
93 | }
94 |
95 | /**
96 | * @param {string} tableName
97 | * @param {string[]} options
98 | * @param {string} whereClause
99 | */
100 | update (tableName, options, whereClause) {
101 | if (arguments.length < 3) {
102 | return Promise.reject(new Error('missed arguments!'))
103 | }
104 |
105 | if (!tableName) {
106 | return Promise.reject(new Error('Table name is not specified!'))
107 | }
108 |
109 | if (!options) {
110 | return Promise.reject(new Error('Record entry is not defined'))
111 | }
112 |
113 | if (!whereClause) {
114 | return Promise.reject(new Error('Where clause is not defined'))
115 | }
116 |
117 | const preparedOptions = _prepareOptions(options)
118 | const setStmtParts = []
119 |
120 | for (let i = 0; i < preparedOptions.keys.length; ++i) {
121 | setStmtParts.push(
122 | `${preparedOptions.keys[i]}=${preparedOptions.values[i]}`
123 | )
124 | }
125 |
126 | const preparedQuery = `UPDATE ${tableName} SET ${setStmtParts.join(
127 | ' , '
128 | )} WHERE ${whereClause}`
129 |
130 | return this._executeSql(preparedQuery, preparedOptions.args)
131 | }
132 |
133 | /**
134 | * @param {string} tableName
135 | * @param {string} whereClause
136 | */
137 | delete (tableName, whereClause) {
138 | if (arguments.length < 2) {
139 | return Promise.reject(new Error('missed arguments!'))
140 | }
141 |
142 | if (!tableName) {
143 | return Promise.reject(new Error('Table name is not specified!'))
144 | }
145 |
146 | if (!whereClause) {
147 | return Promise.reject(new Error('Where clause is not defined'))
148 | }
149 |
150 | const preparedQuery = `DELETE FROM ${tableName} WHERE ${whereClause}`
151 |
152 | return this._executeSql(preparedQuery)
153 | }
154 |
155 | /**
156 | * @param {string} tableName
157 | */
158 | drop (tableName) {
159 | if (!tableName) {
160 | return Promise.reject(new Error('Table name is not specified!'))
161 | }
162 |
163 | const preparedQuery = `DROP TABLE ${tableName}`
164 | return this._executeSql(preparedQuery)
165 | }
166 |
167 | /**
168 | * @param {string} tableName
169 | */
170 | dropBlobTable (tableName) {
171 | if (!tableName) {
172 | return Promise.reject(new Error('Table name is not specified!'))
173 | }
174 |
175 | const preparedQuery = `DROP BLOB TABLE ${tableName}`
176 | return this._executeSql(preparedQuery)
177 | }
178 |
179 | /**
180 | * @param {string} sql statement
181 | * @param {array} args (optional)
182 | */
183 | execute (sql, args = []) {
184 | return this._executeSql(sql, args)
185 | }
186 |
187 | executeBulk (sql, bulkArgs = []) {
188 | return this._executeSql(sql, bulkArgs, true)
189 | }
190 |
191 | /**
192 | * @param {string} tableName
193 | * @param {string} buffer
194 | */
195 | insertBlob (tableName, buffer) {
196 | return new Promise((resolve, reject) => {
197 | const hashCode = crypto
198 | .createHash('sha1')
199 | .update(buffer, 'binary')
200 | .digest('hex')
201 |
202 | const callback = function (response) {
203 | let body = ''
204 |
205 | response.on('data', data => {
206 | body += data.toString()
207 | return undefined
208 | })
209 |
210 | response.on('error', err => {
211 | return reject(err)
212 | })
213 |
214 | response.on('end', () => {
215 | // if the object already exists CRATE returns 409 status code
216 | if (response.statusCode === 409) {
217 | return reject(new Error('error 409: already exists'))
218 | }
219 |
220 | // if everything is alreight CRATE returns '201 created' status
221 | if (response.statusCode > 299) {
222 | return reject(
223 | new Error(
224 | `error HTTP status code: ${response.statusCode} ${body}`
225 | )
226 | )
227 | }
228 |
229 | return resolve(hashCode)
230 | })
231 | }
232 |
233 | const req = this.createBlobWriteStream(tableName, hashCode, callback)
234 |
235 | try {
236 | req.write(buffer)
237 | req.end()
238 | } catch (ex) {
239 | return reject(ex)
240 | }
241 | })
242 | }
243 |
244 | /**
245 | * @param {string} tableName
246 | * @param {string} filename
247 | */
248 | insertBlobFile (tableName, filename) {
249 | return new Promise((resolve, reject) => {
250 | fs.readFile(filename, (err, data) => {
251 | if (err) {
252 | return reject(err)
253 | }
254 |
255 | this.insertBlob(tableName, data)
256 | .then(res => resolve(res))
257 | .catch(err => reject(err))
258 | })
259 | })
260 | }
261 |
262 | /**
263 | * @param {string} tableName
264 | * @param {string} hashKey
265 | */
266 | getBlob (tableName, hashKey) {
267 | return new Promise((resolve, reject) => {
268 | const callback = function (response) {
269 | const buffer = []
270 |
271 | response.on('data', chunk => {
272 | buffer.push(chunk)
273 | })
274 |
275 | response.on('error', err => {
276 | return reject(err)
277 | })
278 |
279 | response.on('end', () => {
280 | return resolve(Buffer.concat(buffer))
281 | })
282 | }
283 |
284 | const reqUrl = `${this.connectionPool.getBlobUrl()}${tableName}/${hashKey}`
285 | http.get(reqUrl, callback)
286 | })
287 | }
288 |
289 | /**
290 | * @param {object} schema like: {person: {name: 'string', age: 'integer'}}
291 | */
292 | create (schema) {
293 | const tableName = Object.keys(schema)[0]
294 | const table = schema[tableName]
295 | const cols = Object.keys(table).map(key => {
296 | return `"${key}" ${table[key]}`
297 | })
298 |
299 | const statement = `CREATE TABLE ${tableName} (${cols})`
300 | return this._executeSql(statement)
301 | }
302 |
303 | /**
304 | * @param {object} schema like: {person: {name: 'string', age: 'integer'}}
305 | */
306 | createIfNotExists (schema) {
307 | const tableName = Object.keys(schema)[0]
308 | const table = schema[tableName]
309 | const cols = Object.keys(table).map(key => {
310 | return `"${key}" ${table[key]}`
311 | })
312 |
313 | const statement = `CREATE TABLE IF NOT EXISTS ${tableName} (${cols})`
314 | return this._executeSql(statement)
315 | }
316 |
317 | /**
318 | * @param {tableName} Name of the BLOB Table
319 | * @param {replicas} Number of replicas
320 | * @param {shards} Number of shards
321 | */
322 | createBlobTable (tableName, replicas, shards) {
323 | const statement = `CREATE BLOB TABLE ${tableName} clustered into ${shards} shards with (number_of_replicas=${replicas})`
324 | return this._executeSql(statement)
325 | }
326 |
327 | /**
328 | * @param {string} sql
329 | * @param {string[]} args
330 | */
331 | _executeSql (sql, args = [], bulk = false) {
332 | return new Promise((resolve, reject) => {
333 | const callback = function (response) {
334 | const data = []
335 |
336 | response.on('data', chunk => {
337 | data.push(chunk)
338 | })
339 |
340 | response.on('error', err => {
341 | return reject(new Error(err))
342 | })
343 |
344 | response.on('end', () => {
345 | let result = {}
346 | let json
347 |
348 | try {
349 | result = JSON.parse(data.join(''))
350 | } catch (ex) {
351 | return reject(new Error(ex))
352 | }
353 |
354 | if (!result.rows) {
355 | /* || /CREATE BLOB/im.test (sql)) */
356 | // workaround CRATE does not return a row when it creates a BLOB
357 | result.rows = []
358 | }
359 |
360 | if (result.error) {
361 | return reject(result.error)
362 | }
363 |
364 | /* jslint nomen: true */
365 | result.__defineGetter__('json', () => {
366 | if (!json) {
367 | json = result.rows.map(e => {
368 | const x = {}
369 | for (let i = 0; i < result.cols.length; ++i) {
370 | if (
371 | result.col_types &&
372 | (result.col_types[i] === crateTypes.TIMESTAMP ||
373 | result.col_types[i] === crateTypes.TIMESTAMPTZ)
374 | ) {
375 | x[result.cols[i]] = new Date(e[i])
376 | } else {
377 | x[result.cols[i]] = e[i]
378 | }
379 | }
380 |
381 | return x
382 | })
383 | }
384 |
385 | return json
386 | })
387 |
388 | /* jslint nomen: false */
389 | return resolve(result)
390 | })
391 | }
392 |
393 | const req = this.connectionPool.getSqlRequest(callback)
394 | // let node know what to do in case of unexpected errors
395 | // before anything has been sent/received.
396 | req.on('error', err => {
397 | console.log('Unexpected connection error', err)
398 | req.end()
399 | return reject(err)
400 | })
401 | let command = {
402 | stmt: sql
403 | }
404 |
405 | if (bulk === true) {
406 | command.bulk_args = args
407 | } else {
408 | command.args = args
409 | }
410 |
411 | command = JSON.stringify(command)
412 |
413 | try {
414 | req.write(command)
415 | req.end()
416 | } catch (ex) {
417 | return reject(ex)
418 | }
419 | })
420 | }
421 |
422 | /**
423 | * @param {string} tableName
424 | * @param {string} streamId - unique ID for the blob
425 | * @param {requestCallback} cb - callback function gor the http request
426 | * @returns http.request object (writeable)
427 | */
428 | createBlobWriteStream (tableName, streamId, cb) {
429 | const options = this.connectionPool.getHttpOptions()
430 | const blobOptions = {
431 | host: options.host,
432 | path: `/_blobs/${tableName}/${streamId}`,
433 | port: options.port,
434 | method: 'PUT',
435 | headers: { 'Conent-Type': 'application/binary' }
436 | }
437 |
438 | return http.request(blobOptions, cb)
439 | }
440 |
441 | getNewInstance () {
442 | return new NodeCrate()
443 | }
444 | }
445 |
446 | module.exports = new NodeCrate()
447 |
--------------------------------------------------------------------------------
/lib/types.js:
--------------------------------------------------------------------------------
1 | 'use strict'
2 |
3 | module.exports = {
4 | // NULL: 0,
5 | UNDEFINED: 0,
6 | NOT_SUPPORTED: 1,
7 | CHAR: 2,
8 | BOOLEAN: 3,
9 | STRING: 4,
10 | IP: 5,
11 | DOUBLE: 6,
12 | FLOAT: 7,
13 | SHORT: 8,
14 | INTEGER: 9,
15 | LONG: 10,
16 | TIMESTAMPTZ: 11,
17 | OBJECT: 12,
18 | GEO_POINT: 13,
19 | GEO_SHAPE: 14,
20 | TIMESTAMP: 15,
21 | REGPROC: 19,
22 | TIME: 20,
23 | OIDVECTOR: 21,
24 | NUMERIC: 22,
25 | REGCLASS: 23,
26 | DATE: 24,
27 | BIT: 25,
28 | JSON: 26,
29 | CHARACTER: 27,
30 | ARRAY: 100,
31 | SET: 101
32 | }
33 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "node-crate",
3 | "version": "3.0.0",
4 | "description": "Node.js SQL-driver for CrateDB (crate.io)",
5 | "main": "lib/index.js",
6 | "scripts": {
7 | "test": "standard --format lib/*.js && mocha -t 20000",
8 | "bundle": "browserify -r ./lib/index.js:node-crate > bundle.js",
9 | "fix": "standard --fix lib/*.js"
10 | },
11 | "repository": {
12 | "type": "git",
13 | "url": "https://github.com/megastef/node-crate.git"
14 | },
15 | "keywords": [
16 | "CRATE",
17 | "CrateDB",
18 | "crate.io",
19 | "sql",
20 | "driver",
21 | "bigdata",
22 | "elasticsearch alternative",
23 | "driver for scalable data store",
24 | "BLOB storage"
25 | ],
26 | "author": "Stefan Thies",
27 | "contributors": [
28 | "Igor Likhomanov",
29 | "Martin Heidegger"
30 | ],
31 | "license": "MIT License",
32 | "bugs": {
33 | "url": "https://github.com/megastef/node-crate/issues"
34 | },
35 | "homepage": "http://megastef.github.io/node-crate/",
36 | "dependencies": {
37 | "type-of-is": ">= 3.3.x"
38 | },
39 | "devDependencies": {
40 | "expect": "^29.5.0",
41 | "mocha": "^10.2.0",
42 | "standard": "^17.0.0"
43 | }
44 | }
45 |
--------------------------------------------------------------------------------
/test/test.js:
--------------------------------------------------------------------------------
1 | 'use strict'
2 |
3 | /* global describe, it */
4 | const {expect} = require('expect')
5 |
6 | const crate = require('../')
7 |
8 | // Why only 50? the default setting in crate ...
9 | // EsThreadPoolExecutor[bulk, queue capacity = 50]
10 | // for more than 50 inserts at once use Bulk insert or increase queue in Crate
11 | const docsToInsert = 50
12 |
13 | crate.connect(process.env.CRATE_URL || 'http://127.0.0.1:4200')
14 |
15 | const blobTableName = 'blob_test_3'
16 | const tableName = 'NodeCrateTest_3'
17 |
18 | describe('#node-crate', function () {
19 | this.slow(15000)
20 |
21 | it('should create blob table', async () => {
22 | const res = await crate.createBlobTable(blobTableName, 0, 1)
23 | expect(res.rowcount).toBe(1)
24 | });
25 |
26 | it('should create table', async () => {
27 | const schema = {}
28 | schema[tableName] = { id: 'integer primary key', title: 'string', numberVal: 'integer' }
29 |
30 | const res = await crate.create(schema)
31 | expect(res.rowcount).toBe(1)
32 | });
33 |
34 | it('should create table if not exists - table exists', async () => {
35 | const schema = {}
36 | schema[tableName] = { id: 'integer primary key', title: 'string', numberVal: 'integer' }
37 |
38 | const res = await crate.createIfNotExists(schema)
39 | expect(res.rowcount).toBe(0)
40 | })
41 |
42 | it('should drop table to create again', async () => {
43 | const res = await crate.drop(tableName)
44 | expect(res.rowcount).toBe(1)
45 | })
46 |
47 | it('should create table if not exists - table does not exist', async () => {
48 | const schema = {}
49 | schema[tableName] = { id: 'integer primary key', title: 'string', numberVal: 'integer' }
50 |
51 | const res = await crate.createIfNotExists(schema)
52 | expect(res.rowcount).toBe(1)
53 | })
54 |
55 | let hashkey = ''
56 |
57 | it('should insert blob', async () => {
58 | const res = await crate.insertBlobFile(blobTableName, './lib/index.js');
59 | expect(res.length).toBe(40);
60 | hashkey = res;
61 | });
62 |
63 | it('should insert', async () => {
64 | const res = await crate.insert(tableName, {
65 | id: '1',
66 | title: 'Title',
67 | numberVal: 42
68 | });
69 | expect(res.rowcount).toBe(1);
70 | });
71 |
72 | it('should insert many', async () => {
73 | let success = 0
74 | let errorReported = false
75 | let longTitle = 'A long title to generate larger chunks ...'
76 |
77 | for (let k = 0; k < 5; ++k) {
78 | longTitle += longTitle
79 | }
80 |
81 | for (let i = 0; i < docsToInsert; ++i) {
82 | await crate.insert(tableName, {
83 | id: i + 100,
84 | title: longTitle,
85 | numberVal: 42
86 | })
87 | .then(() => {
88 | success++;
89 | })
90 | .catch((err) => {
91 | throw err;
92 | });
93 | }
94 | expect(success).toBe(docsToInsert);
95 | });
96 |
97 | it('should insert bulk documents', async () => {
98 | const title = 'A title';
99 | const bulkArgs = [];
100 |
101 | for (let i = 0; i < docsToInsert; ++i) {
102 | bulkArgs[i] = [i + 1000, title, 42];
103 | }
104 |
105 | const res = await crate.executeBulk(`INSERT INTO ${tableName} ("id","title","numberVal") Values (?, ?, ?)`, bulkArgs);
106 | expect(res.results.length).toBe(docsToInsert);
107 | });
108 |
109 | it('should select', async () => {
110 | await crate.execute(`REFRESH TABLE ${tableName}`);
111 | const res = await crate.execute(`SELECT * FROM ${tableName} limit ${docsToInsert}`);
112 | expect(res.rowcount).toBe(docsToInsert);
113 | });
114 |
115 | it('should update', async () => {
116 | const res = await crate.update(tableName, {
117 | title: 'TitleNew'
118 | }, 'id=1');
119 | expect(res.rowcount).toBe(1);
120 | });
121 |
122 | it('should select after update', async () => {
123 | await crate.execute(`REFRESH TABLE ${tableName}`);
124 | const res = await crate.execute(`SELECT * FROM ${tableName} where id=1 limit 100`);
125 | expect(res.json[0].title).toBe('TitleNew');
126 | expect(res.json[0].numberVal).toBe(42);
127 | });
128 |
129 | it('should get blob', async () => {
130 | const res = await crate.getBlob('blobtest', hashkey);
131 | expect(res instanceof Buffer).toBe(true);
132 | });
133 |
134 | it('should delete', async () => {
135 | const res = await crate.delete(tableName, 'id=1');
136 | await crate.execute(`REFRESH TABLE ${tableName}`);
137 | expect(res.rowcount).toBe(1);
138 | });
139 |
140 | it('should drop table', async () => {
141 | const res = await crate.drop(tableName);
142 | expect(res.rowcount).toBe(1);
143 | });
144 |
145 | it('should drop blob table', async () => {
146 | const res = await crate.dropBlobTable(blobTableName);
147 | expect(res.rowcount).toBe(1);
148 | });
149 |
150 | it('should fail to drop blob table with empty name', async () => {
151 | try {
152 | await crate.dropBlobTable();
153 | } catch (err) {
154 | expect(err.message).toBe('Table name is not specified!');
155 | }
156 | });
157 | })
158 |
--------------------------------------------------------------------------------