├── package.json ├── LICENSE ├── .gitignore ├── README.md └── index.js /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloudflare-workers-kv", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/bitquant/cloudflare-workers-kv.git" 12 | }, 13 | "author": "bitquant", 14 | "license": "MIT", 15 | "bugs": { 16 | "url": "https://github.com/bitquant/cloudflare-workers-kv/issues" 17 | }, 18 | "homepage": "https://github.com/bitquant/cloudflare-workers-kv#readme", 19 | "dependencies": { 20 | "uuid": "3.3.3" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 bitquant 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # next.js build output 61 | .next 62 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cloudflare-workers-kv 2 | 3 | Use Cloudflare Workers KV in your local environment or within a Cloudflare Workers environment. The package also provides support for handling values larger than the 25 MB limit by breaking the value into smaller chunks. 4 | 5 | ## Install 6 | ``` 7 | $ npm install cloudflare-workers-kv --save 8 | ``` 9 | 10 | ## Usage 11 | ```javascript 12 | var kv = require('cloudflare-workers-kv'); 13 | 14 | // These dependencies are needed when running in Node 15 | global.fetch = require('node-fetch'); 16 | var util = require('util'); 17 | global.TextEncoder = util.TextEncoder; 18 | global.TextDecoder = util.TextDecoder; 19 | 20 | (async () => { 21 | kv.init({ 22 | variableBinding: '', 23 | namespaceId: '', 24 | accountId: '', 25 | email: '', 26 | apiKey: '' 27 | }); 28 | 29 | // Write to the KV store 30 | await kv.put('test-kv-key', 'test-key-value'); 31 | 32 | // Write to the KV store with an expiration 33 | await kv.put('test-kv-key-exp', 'test-key-value', { expirationTtl: 60 }); 34 | 35 | // Read from the KV store 36 | var data = await kv.get('test-kv-key'); 37 | 38 | // Delete from the KV store 39 | await kv.del('test-kv-key'); 40 | 41 | // Bulk write to the KV store - does not support values larger than 25 MB 42 | await kv.putMulti([ 43 | { key: 'key-abc', value: 'value-abc' }, 44 | { key: 'key-xyz', value: 'value-xyz' }, 45 | { key: 'key-123', value: 'value-123', expirationTtl: 60 } 46 | ]) 47 | 48 | // List all keys 49 | await kv.list(); 50 | 51 | // List keys with cursor, prefix and result limit 52 | await kv.list({prefix: "abc", limit: 75, cursor: "2d840f03-df70-4d93-afe4-5f83856f6214"}) 53 | })(); 54 | 55 | ``` 56 | 57 | ## Multi Environment Support 58 | The library can be used within a worker running in Cloudflare as well as within any local test environment. When running in a local test environment the library uses the Cloudflare KV rest API. 59 | 60 | 61 | ## Large Value Support 62 | If a value to be written exceeds the 25 MB Cloudflare limit, the value will be broken into chunks and stored as multiple values. When reading back the value multiple reads will occur in parallel and the value will be pieced back together for use. When deleting a large value the library will take care of deleting all chunks that were created. If a key with a large value is overwritten with a new value the library will provide a "clean-up" id. It takes up to 60 seconds for KV changes to be written to all data centers. In order to maintain value consistency the old chunks are not removed immediately after being overwritten. Use `kv.clean('')` to remove the old chunks. It is recommended to call `clean` no earlier than 60 seconds after receiving a clean-up id. 63 | 64 | *NOTE*: kv.putMulti() does not support values larger than 25 MB! 65 | 66 | ## License 67 | MIT license; see [LICENSE](./LICENSE). 68 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var uuidv4 = require('uuid/v4'); 2 | 3 | var ACCOUNT_ID = null; 4 | var EMAIL = null; 5 | var API_KEY = null; 6 | var NAMESPACE_ID = null; 7 | var BINDING = null; 8 | 9 | var BASE_PATH = 'https://api.cloudflare.com/client/v4/accounts' 10 | var BLOCK_SIZE = 25000000 // KV max size 11 | var BLOCK_REGEX = /^id=[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12};length=[0-9]{1,}$/ 12 | 13 | 14 | async function init(config) { 15 | 16 | BINDING = config.variableBinding; 17 | NAMESPACE_ID = config.namespaceId; 18 | ACCOUNT_ID = config.accountId; 19 | EMAIL = config.email; 20 | API_KEY = config.apiKey; 21 | } 22 | 23 | async function getWithRestApi(key, type) { 24 | 25 | const response = await fetch(`${BASE_PATH}/${ACCOUNT_ID}/storage/kv/namespaces/${NAMESPACE_ID}/values/${key}`, { 26 | headers: { 27 | 'X-Auth-Email': EMAIL, 28 | 'X-Auth-Key': API_KEY 29 | } 30 | }); 31 | 32 | if (!response.ok) { 33 | return null; 34 | } 35 | 36 | if (type === 'text' || type === undefined) { 37 | return response.text(); 38 | } 39 | else if (type === 'json') { 40 | return response.json(); 41 | } 42 | else if (type === 'arrayBuffer') { 43 | return response.arrayBuffer(); 44 | } 45 | else if (type === 'stream') { 46 | return response.body; 47 | } 48 | else { 49 | throw new Error(`error getting value for key ${key}, unsupported type: ${type}`) 50 | } 51 | } 52 | 53 | function parseBlockMeta(value) { 54 | let blockId = value.slice(3, 39); 55 | let blockCount = parseInt(value.slice(47), 10); 56 | return { blockId, blockCount }; 57 | } 58 | 59 | function getBlockMeta(blockId, blockCount) { 60 | return `id=${blockId};length=${blockCount}`; 61 | } 62 | 63 | function getBlockKey(blockId, blockIndex) { 64 | return `id=${blockId};index=${blockIndex}`; 65 | } 66 | 67 | async function get(key, type) { 68 | 69 | let arrayBuffer = await getKV(key, 'arrayBuffer'); 70 | 71 | if (arrayBuffer === null) { 72 | return null; 73 | } 74 | 75 | let stringValue = new TextDecoder().decode(arrayBuffer); 76 | 77 | if (stringValue.search(BLOCK_REGEX) === -1) { 78 | 79 | if (type === 'text' || type === undefined) { 80 | return stringValue; 81 | } 82 | if (type === 'json') { 83 | return JSON.parse(stringValue) 84 | } 85 | if (type === 'arrayBuffer') { 86 | return arrayBuffer; 87 | } 88 | if (type === 'stream') { 89 | return new Response(arrayBuffer).body; 90 | } 91 | 92 | throw new Error(`error getting value for key ${key}, unsupported type: ${type}`) 93 | } 94 | 95 | let { blockId, blockCount } = parseBlockMeta(stringValue); 96 | let promiseList = []; 97 | 98 | for (let blockIndex = 0; blockIndex < blockCount; blockIndex++) { 99 | let blockKey = getBlockKey(blockId, blockIndex); 100 | let blockPromise = getKV(blockKey, 'arrayBuffer') 101 | promiseList.push(blockPromise) 102 | } 103 | 104 | let blockList = await Promise.all(promiseList); 105 | let byteArraySize = 0; 106 | 107 | for (let blockData of blockList) { 108 | if (blockData === null) { 109 | let err = new Error(`key '${key}' has missing data blocks and needs deletion`); 110 | err.blockRecord = stringValue; 111 | throw err; 112 | } 113 | byteArraySize += blockData.byteLength; 114 | } 115 | 116 | let resultArray = new Uint8Array(byteArraySize); 117 | let offset = 0; 118 | for (let blockData of blockList) { 119 | resultArray.set(new Uint8Array(blockData), offset); 120 | offset += blockData.byteLength; 121 | } 122 | 123 | let finalValue; 124 | 125 | if (type === 'text' || type === undefined || type === 'json') { 126 | finalValue = new TextDecoder().decode(resultArray); 127 | if (type === 'json') { 128 | finalValue = JSON.parse(finalValue) 129 | } 130 | } 131 | else if (type === 'arrayBuffer') { 132 | finalValue = resultArray.buffer; 133 | } 134 | else if (type === 'stream') { 135 | finalValue = new Response(resultArray.buffer).body; 136 | } 137 | else { 138 | throw new Error(`error getting large value for key ${key}, unsupported type: ${type}`) 139 | } 140 | 141 | return finalValue; 142 | } 143 | 144 | async function putWithRestApi(key, value, params) { 145 | 146 | let query = ''; 147 | 148 | if (params !== undefined) { 149 | if (params.expiration !== undefined) { 150 | query += `?expiration=${params.expiration}` 151 | } 152 | if (params.expirationTtl !== undefined) { 153 | query += `${query.length > 0 ? '&' : '?'}expiration_ttl=${params.expirationTtl}` 154 | } 155 | } 156 | 157 | const response = await fetch(`${BASE_PATH}/${ACCOUNT_ID}/storage/kv/namespaces/${NAMESPACE_ID}/values/${key}${query}`, { 158 | headers: { 159 | 'X-Auth-Email': EMAIL, 160 | 'X-Auth-Key': API_KEY 161 | }, 162 | method: 'PUT', 163 | body: value 164 | }); 165 | 166 | if (!response.ok) { 167 | throw new Error(`${NAMESPACE_ID}:${key} not set to ${value} status: ${response.status}`); 168 | } 169 | 170 | let body = await response.json(); 171 | 172 | if (body.success !== true) { 173 | throw new Error(`${NAMESPACE_ID}:${key} not set to ${value} success: ${body.success}`); 174 | } 175 | 176 | return undefined; 177 | } 178 | 179 | async function put(key, value, params) { 180 | 181 | let oldValue = await getKV(key, 'text'); 182 | let oldBlock = undefined; 183 | 184 | if (oldValue !== null && oldValue.search(BLOCK_REGEX) === 0) { 185 | oldBlock = oldValue; 186 | } 187 | 188 | let encoded = null; 189 | 190 | if (typeof value === 'string') { 191 | encoded = new TextEncoder().encode(value); 192 | } 193 | else if (value instanceof ArrayBuffer) { 194 | encoded = new Uint8Array(value) 195 | } 196 | else if (value.buffer instanceof ArrayBuffer) { // ArrayBufferView 197 | encoded = new Uint8Array(value.buffer) 198 | } 199 | else { 200 | encoded = new Uint8Array(await new Response(value).arrayBuffer()) 201 | } 202 | 203 | if (encoded.length <= BLOCK_SIZE) { 204 | await putKV(key, encoded, params); 205 | return oldBlock; 206 | } 207 | 208 | let blockList = []; 209 | let blocks = Math.floor(encoded.length / BLOCK_SIZE); 210 | let lastBlock = (encoded.length % BLOCK_SIZE) > 0 ? 1 : 0; 211 | let totalBlocks = blocks + lastBlock; 212 | 213 | for (let i = 0; i < totalBlocks; i++) { 214 | let startIndex = i * BLOCK_SIZE; 215 | let endIndex = startIndex + BLOCK_SIZE; 216 | let block = encoded.slice(startIndex, endIndex); 217 | blockList.push(block); 218 | } 219 | 220 | let blockId = uuidv4(); 221 | let blockIndex = 0; 222 | 223 | if (params !== undefined) { 224 | // Blocks need to expire after the header block expires so +600s 225 | var blockParams = Object.assign({}, params); 226 | if (params.expiration !== undefined) { 227 | blockParams.expiration += 600; 228 | } 229 | if (params.expirationTtl !== undefined) { 230 | blockParams.expirationTtl += 600; 231 | } 232 | } 233 | 234 | for (let block of blockList) { 235 | let blockKey = getBlockKey(blockId, blockIndex); 236 | try { 237 | await putKV(blockKey, block.buffer, blockParams); 238 | } 239 | catch (ex) { 240 | throw new Error(`${blockKey} block put error: ${ex.message}`) 241 | } 242 | blockIndex++; 243 | } 244 | 245 | let blockMeta = getBlockMeta(blockId, blockList.length) 246 | await putKV(key, blockMeta, params); 247 | 248 | return oldBlock; 249 | } 250 | 251 | async function delWithRestApi(key) { 252 | 253 | const response = await fetch(`${BASE_PATH}/${ACCOUNT_ID}/storage/kv/namespaces/${NAMESPACE_ID}/values/${key}`, { 254 | headers: { 255 | 'X-Auth-Email': EMAIL, 256 | 'X-Auth-Key': API_KEY 257 | }, 258 | method: 'DELETE' 259 | }); 260 | 261 | // Check if key not found 262 | if (response.status === 404) { 263 | return false; // key does not exist 264 | } 265 | 266 | if (!response.ok) { 267 | throw new Error(`${NAMESPACE_ID}:${key} not deleted, status: ${response.status}`); 268 | } 269 | 270 | let body = await response.json(); 271 | 272 | if (body.success !== true) { 273 | throw new Error(`${NAMESPACE_ID}:${key} not deleted, success: ${body.success}`); 274 | } 275 | 276 | return true; // key deleted 277 | } 278 | 279 | async function delWithNameSpace(key) { 280 | 281 | try { 282 | await self[BINDING].delete(key); 283 | return true; // key deleted 284 | } 285 | catch (ex) { 286 | if (ex.message.includes('404')) { 287 | return false; // key does not exist 288 | } 289 | throw ex; 290 | } 291 | } 292 | 293 | async function del(key) { 294 | 295 | let value = await getKV(key); 296 | 297 | if (value === null) { 298 | return false; 299 | } 300 | 301 | if (value.search(BLOCK_REGEX) === -1) { 302 | return delKV(key); 303 | } 304 | 305 | let { blockId, blockCount } = parseBlockMeta(value); 306 | 307 | for (let blockIndex = 0; blockIndex < blockCount; blockIndex++) { 308 | let blockKey = getBlockKey(blockId, blockIndex); 309 | await delKV(blockKey); 310 | } 311 | 312 | return delKV(key); 313 | } 314 | 315 | async function clean(block) { 316 | 317 | let { blockId, blockCount } = parseBlockMeta(block); 318 | 319 | for (let blockIndex = 0; blockIndex < blockCount; blockIndex++) { 320 | let blockKey = getBlockKey(blockId, blockIndex); 321 | await delKV(blockKey); 322 | } 323 | } 324 | 325 | async function putMultiWithRestApi(keyValuePairs) { 326 | 327 | keyValuePairs.map(function(kv) { 328 | if (kv.expirationTtl !== undefined) { 329 | kv.expiration_ttl = kv.expirationTtl; 330 | delete kv.expirationTtl; 331 | } 332 | return kv; 333 | }) 334 | 335 | const response = await fetch(`${BASE_PATH}/${ACCOUNT_ID}/storage/kv/namespaces/${NAMESPACE_ID}/bulk`, { 336 | headers: { 337 | 'X-Auth-Email': EMAIL, 338 | 'X-Auth-Key': API_KEY, 339 | 'Content-Type': 'application/json' 340 | }, 341 | method: 'PUT', 342 | body: JSON.stringify(keyValuePairs) 343 | }); 344 | 345 | if (!response.ok) { 346 | throw new Error(`${NAMESPACE_ID}:multiple keys not set status: ${response.status}`); 347 | } 348 | 349 | let body = await response.json(); 350 | 351 | if (body.success !== true) { 352 | throw new Error(`${NAMESPACE_ID}:multiple keys not set success: ${body.success}`); 353 | } 354 | 355 | return undefined; 356 | } 357 | 358 | async function putMulti(keyValuePairs) { 359 | 360 | return putMultiKV(keyValuePairs); 361 | } 362 | 363 | async function listWithRestApi(params) { 364 | 365 | let query = ''; 366 | 367 | if (params !== undefined) { 368 | 369 | if (params.cursor !== undefined) { 370 | query += `?cursor=${params.cursor}` 371 | } 372 | 373 | if (params.prefix !== undefined) { 374 | query += `${query.length > 0 ? '&' : '?'}prefix=${params.prefix}` 375 | } 376 | 377 | if (params.limit !== undefined) { 378 | query += `${query.length > 0 ? '&' : '?'}limit=${params.limit}` 379 | } 380 | } 381 | 382 | const response = await fetch(`${BASE_PATH}/${ACCOUNT_ID}/storage/kv/namespaces/${NAMESPACE_ID}/keys${query}`, { 383 | headers: { 384 | 'X-Auth-Email': EMAIL, 385 | 'X-Auth-Key': API_KEY 386 | } 387 | }); 388 | 389 | if (!response.ok) { 390 | throw new Error(`${NAMESPACE_ID}: unable to list keys status: ${response.status}`); 391 | } 392 | 393 | let body = await response.json(); 394 | 395 | if (body.success !== true) { 396 | throw new Error(`${NAMESPACE_ID}: list keys not successful`); 397 | } 398 | 399 | return { 400 | count: body.result_info.count, 401 | cursor: body.result_info.cursor, 402 | keys: body.result 403 | }; 404 | } 405 | 406 | async function list(params) { 407 | 408 | return listKV(params) 409 | } 410 | 411 | var getKV = (typeof self === 'undefined') ? getWithRestApi : (key, type) => self[BINDING].get(key, type); 412 | var putKV = (typeof self === 'undefined') ? putWithRestApi : (key, value, params) => self[BINDING].put(key, value, params); 413 | var delKV = (typeof self === 'undefined') ? delWithRestApi : delWithNameSpace; 414 | var putMultiKV = putMultiWithRestApi; 415 | var listKV = listWithRestApi; 416 | 417 | 418 | exports.init = init; 419 | exports.get = get; 420 | exports.put = put; 421 | exports.del = del; 422 | exports.clean = clean; 423 | exports.putMulti = putMulti; 424 | exports.list = list; 425 | --------------------------------------------------------------------------------