├── .gitignore ├── .npmignore ├── .travis.yml ├── .vscode └── launch.json ├── LICENSE ├── README.md ├── SECURITY.md ├── package.json ├── src ├── CordovaNativeSqliteProvider.ts ├── FullTextSearchHelpers.ts ├── InMemoryProvider.ts ├── IndexedDbProvider.ts ├── NoSqlProvider.ts ├── NoSqlProviderUtils.ts ├── NodeSqlite3DbProvider.ts ├── Promise.d.ts ├── Promise.extensions.ts ├── SqlProviderBase.ts ├── StoreHelpers.ts ├── TransactionLockHelper.ts ├── WebSqlProvider.ts ├── defer.ts ├── get-window.ts └── tests │ └── NoSqlProviderTests.ts ├── test.html ├── tsconfig.json ├── tslint.json ├── webpack.config.js └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /src/.vs 3 | /src/bin 4 | /src/obj 5 | *.user 6 | /NoSQLProviderTestsPack.js 7 | /dist 8 | .DS_Store 9 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /src/.vs 3 | /src/bin 4 | /src/obj 5 | *.user 6 | /src/typings 7 | /NoSQLProviderTestsPack.js 8 | /.vscode 9 | test.html 10 | tsconfig.json 11 | webpack.config.js 12 | README.md 13 | .DS_Store -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "node" 4 | cache: 5 | directories: 6 | - "node_modules" 7 | script: npm run ci-test 8 | dist: xenial 9 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Debug Mocha Tests", 11 | "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", 12 | "args": [ 13 | "--timeout", 14 | "5000", 15 | "--colors", 16 | "${workspaceFolder}/dist/tests/NoSqlProviderTests.js" 17 | ], 18 | "console": "integratedTerminal", 19 | "internalConsoleOptions": "neverOpen" 20 | } 21 | ] 22 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Microsoft Corporation 2 | 3 | All rights reserved. 4 | 5 | The MIT License (MIT) 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all 15 | copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NoSQLProvider [![Build Status](https://travis-ci.org/Microsoft/NoSQLProvider.svg?branch=master)](https://travis-ci.org/Microsoft/NoSQLProvider) 2 | We developed NoSQLProvider after needing a simplified interface to NoSQL-style object storage/retrieval that worked not only across all browsers, but also for native app development (initially Cordova-based, later moving to React Native.) Across the browsers, this required unifying WebSQL and IndexedDB, with the nuances of all the different IndexedDB issues (on IE, mostly, with not properly supporting compound keys and multi-entry indexes, and Safari having basically completely broken keyspace support.) On native platforms and NodeJS, we fully wrap several different SQLite-based providers, depending on the platform. We also have built a fully in-memory database provider that has no persistence but supports fully transactional semantics, for a fallback in situations where you don't want persistence (ephemeral logins, etc.) 3 | 4 | 5 | At this point, known feature work is complete and we are mostly just fixing bugs on the platform moving forward. If you have feature requests, feel free to file them, just make sure that you have behavior determined across all providers as part of your request, since all features need to be able to work across all the different providers. 6 | 7 | # Examples 8 | The only current full example is available as part of the ReactXP samples, the [TodoList](https://github.com/Microsoft/reactxp/tree/master/samples/TodoList) sample app. If you pick up NoSQLProvider and use it in your open source application, please let us know so we can add more example links here! 9 | 10 | # Providers/Platforms/Support 11 | 12 | We will soon list all of the providers available out of the box here, and list what platforms they can be used on, with any nuances of that platform. 13 | 14 | # Usage 15 | 16 | Coming soon. 17 | 18 | ## Compiling 19 | ### Source 20 | ```bash 21 | yarn install 22 | yarn build 23 | ``` 24 | ### Tests 25 | ```bash 26 | yarn install 27 | yarn webtest 28 | ``` 29 | 30 | ## Testing 31 | 1. Compile tests 32 | 1. Open test.html in browser 33 | 1. You can add `?grep=foo` to the URL to run only tests containing 'foo' in the name 34 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nosqlprovider", 3 | "version": "0.6.23", 4 | "description": "A cross-browser/platform indexeddb-like client library", 5 | "author": "David de Regt ", 6 | "scripts": { 7 | "ci-test": "npm run build && npm run test", 8 | "test": "mocha dist/tests/NoSqlProviderTests.js --timeout 5000", 9 | "webtest": "webpack --watch", 10 | "build": "npm run tslint && tsc", 11 | "tslint": "tslint --project tsconfig.json -r tslint.json -r ./node_modules/tslint-microsoft-contrib --fix || true" 12 | }, 13 | "main": "dist/NoSqlProvider.js", 14 | "sideEffects": false, 15 | "dependencies": { 16 | "@types/lodash": "^4.14.149", 17 | "@types/sqlite3": "^3.1.6", 18 | "core-js": "^3.6.5", 19 | "lodash": "^4.17.15", 20 | "regexp-i18n": "^1.3.2" 21 | }, 22 | "devDependencies": { 23 | "@types/mocha": "5.2.7", 24 | "@types/node": "13.1.7", 25 | "@types/sinon": "^7.5.1", 26 | "awesome-typescript-loader": "^5.2.1", 27 | "mocha": "^7.0.0", 28 | "sinon": "^8.1.0", 29 | "sqlite3": "^4.1.1", 30 | "tslint": "^5.20.1", 31 | "tslint-microsoft-contrib": "^6.2.0", 32 | "typescript": "^4.0", 33 | "webpack": "^4.41.5", 34 | "webpack-cli": "^3.3.10" 35 | }, 36 | "repository": { 37 | "type": "git", 38 | "url": "https://github.com/Microsoft/NoSQLProvider" 39 | }, 40 | "bugs": { 41 | "url": "https://github.com/Microsoft/NoSQLProvider/issues" 42 | }, 43 | "typings": "dist/NoSqlProvider.d.ts", 44 | "typescript": { 45 | "definition": "dist/NoSqlProvider.d.ts" 46 | }, 47 | "keywords": [ 48 | "nosql", 49 | "indexeddb", 50 | "websql", 51 | "sqlite", 52 | "browser", 53 | "react native" 54 | ], 55 | "license": "MIT" 56 | } 57 | -------------------------------------------------------------------------------- /src/CordovaNativeSqliteProvider.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * CordovaNativeSqliteProvider.ts 3 | * Author: David de Regt 4 | * Copyright: Microsoft 2015 5 | * 6 | * NoSqlProvider provider setup for cordova-native-sqlite, a cordova plugin backed by sqlite3. 7 | * Also works for react-native-sqlite-storage, since it's based on the same bindings, just make sure to pass in an instance 8 | * of the plugin into the constructor to be used, since window.sqlitePlugin won't exist. 9 | */ 10 | 11 | import { extend } from 'lodash'; 12 | 13 | import { DbSchema } from './NoSqlProvider'; 14 | import { SQLTransactionErrorCallback, SQLResultSet, SQLVoidCallback, SqlProviderBase, SqlTransaction, SqliteSqlTransaction }from './SqlProviderBase'; 15 | import { TransactionLockHelper, TransactionToken } from './TransactionLockHelper'; 16 | import { IDeferred, defer } from './defer'; 17 | // Extending interfaces that should be in lib.d.ts but aren't for some reason. 18 | declare global { 19 | interface Window { 20 | sqlitePlugin: any; 21 | } 22 | } 23 | 24 | // declare enum SQLErrors { 25 | // UNKNOWN_ERR = 0, 26 | // DATABASE_ERR = 1, 27 | // VERSION_ERR = 2, 28 | // TOO_LARGE_ERR = 3, 29 | // QUOTA_ERR = 4, 30 | // SYNTAX_ERR = 5, 31 | // CONSTRAINT_ERR = 6, 32 | // TIMEOUT_ERR = 7 33 | // } 34 | 35 | interface SQLError { 36 | code: number; 37 | message: string; 38 | } 39 | 40 | interface SQLStatementCallback { 41 | (transaction: SQLTransaction, resultSet: SQLResultSet): void; 42 | } 43 | 44 | interface SQLStatementErrorCallback { 45 | (transaction: SQLTransaction, error: SQLError): void; 46 | } 47 | 48 | interface SQLTransaction { 49 | executeSql(sqlStatement: string, args?: any[], callback?: SQLStatementCallback, errorCallback?: SQLStatementErrorCallback): void; 50 | } 51 | 52 | export type SqliteSuccessCallback = () => void; 53 | export type SqliteErrorCallback = (e: Error) => void; 54 | 55 | export interface SqlitePluginDbOptionalParams { 56 | createFromLocation?: number; 57 | androidDatabaseImplementation?: number; 58 | // Database encryption pass phrase 59 | key?: string; 60 | } 61 | 62 | export interface SqlitePluginDbParams extends SqlitePluginDbOptionalParams { 63 | name: string; 64 | location: number; 65 | } 66 | 67 | export interface CordovaTransactionCallback { 68 | (transaction: CordovaTransaction): void; 69 | } 70 | 71 | export interface SqliteDatabase { 72 | openDBs: string[]; 73 | transaction(callback?: CordovaTransactionCallback, errorCallback?: SQLTransactionErrorCallback, 74 | successCallback?: SQLVoidCallback): void; 75 | readTransaction(callback?: CordovaTransactionCallback, errorCallback?: SQLTransactionErrorCallback, 76 | successCallback?: SQLVoidCallback): void; 77 | open(success: SqliteSuccessCallback, error: SqliteErrorCallback): void; 78 | close(success: SqliteSuccessCallback, error: SqliteErrorCallback): void; 79 | executeSql(statement: string, params?: any[], success?: SQLStatementCallback, 80 | error?: SQLStatementErrorCallback): void; 81 | } 82 | 83 | export interface SqlitePlugin { 84 | openDatabase(dbInfo: SqlitePluginDbParams, success?: SqliteSuccessCallback, error?: SqliteErrorCallback): SqliteDatabase; 85 | deleteDatabase(dbInfo: SqlitePluginDbParams, success?: SqliteSuccessCallback, error?: SqliteErrorCallback): void; 86 | sqliteFeatures: { isSQLitePlugin: boolean }; 87 | } 88 | 89 | export interface CordovaTransaction extends SQLTransaction { 90 | abort(err?: any): void; 91 | } 92 | 93 | export class CordovaNativeSqliteProvider extends SqlProviderBase { 94 | private _lockHelper: TransactionLockHelper|undefined; 95 | 96 | // You can use the openOptions object to pass extra optional parameters like androidDatabaseImplementation to the open command 97 | constructor(private _plugin: SqlitePlugin = window.sqlitePlugin, private _openOptions: SqlitePluginDbOptionalParams = {}) { 98 | super(true); 99 | } 100 | 101 | private _db: SqliteDatabase|undefined; 102 | 103 | private _dbParams: SqlitePluginDbParams|undefined; 104 | private _closingDefer: IDeferred|undefined; 105 | 106 | open(dbName: string, schema: DbSchema, wipeIfExists: boolean, verbose: boolean): Promise { 107 | super.open(dbName, schema, wipeIfExists, verbose); 108 | this._lockHelper = new TransactionLockHelper(schema, true); 109 | 110 | if (!this._plugin || !this._plugin.openDatabase) { 111 | return Promise.reject('No support for native sqlite in this browser'); 112 | } 113 | 114 | if (typeof (navigator) !== 'undefined' && navigator.userAgent && navigator.userAgent.indexOf('Mobile Crosswalk') !== -1) { 115 | return Promise.reject('Android NativeSqlite is broken, skipping'); 116 | } 117 | 118 | this._dbParams = extend({ 119 | name: dbName + '.db', 120 | location: 2 121 | }, this._openOptions); 122 | 123 | const task = defer(); 124 | this._db = this._plugin.openDatabase(this._dbParams, () => { 125 | task.resolve(void 0); 126 | }, (err: any) => { 127 | task.reject('Couldn\'t open database: ' + dbName + ', error: ' + JSON.stringify(err)); 128 | }); 129 | 130 | return task.promise.then(() => { 131 | return this._ourVersionChecker(wipeIfExists); 132 | }).catch(err => { 133 | return Promise.reject('Version check failure. Couldn\'t open database: ' + dbName + 134 | ', error: ' + JSON.stringify(err)); 135 | }); 136 | } 137 | 138 | close(): Promise { 139 | if (!this._db) { 140 | return Promise.reject('Database already closed'); 141 | } 142 | 143 | return this._lockHelper!!!.closeWhenPossible().then(() => { 144 | let def = defer(); 145 | this._db!!!.close(() => { 146 | this._db = undefined; 147 | def.resolve(void 0); 148 | }, (err: any) => { 149 | def.reject(err); 150 | }); 151 | return def.promise; 152 | }); 153 | } 154 | 155 | protected _deleteDatabaseInternal(): Promise { 156 | if (!this._plugin || !this._plugin.deleteDatabase) { 157 | return Promise.reject('No support for deleting'); 158 | } 159 | let task = defer(); 160 | this._plugin.deleteDatabase(this._dbParams!!!, () => { 161 | task.resolve(void 0); 162 | }, err => { 163 | task.reject('Couldn\'t delete the database ' + this._dbName + ', error: ' + JSON.stringify(err)); 164 | }); 165 | return task.promise; 166 | } 167 | 168 | openTransaction(storeNames: string[], writeNeeded: boolean): Promise { 169 | if (!this._db) { 170 | return Promise.reject('Can\'t openTransation, Database closed'); 171 | } 172 | 173 | if (this._closingDefer) { 174 | return Promise.reject('Currently closing provider -- rejecting transaction open'); 175 | } 176 | 177 | return this._lockHelper!!!.openTransaction(storeNames, writeNeeded).then(transToken => { 178 | const deferred = defer(); 179 | 180 | let ourTrans: SqliteSqlTransaction; 181 | (writeNeeded ? this._db!!!.transaction : this._db!!!.readTransaction).call(this._db, (trans: CordovaTransaction) => { 182 | ourTrans = new CordovaNativeSqliteTransaction(trans, this._lockHelper!!!, transToken, this._schema!!!, this._verbose!!!, 183 | 999, this._supportsFTS3); 184 | deferred.resolve(ourTrans); 185 | }, (err: SQLError) => { 186 | if (ourTrans) { 187 | ourTrans.internal_markTransactionClosed(); 188 | this._lockHelper!!!.transactionFailed(transToken, 'CordovaNativeSqliteTransaction Error: ' + err.message); 189 | } else { 190 | // We need to reject the transaction directly only in cases when it never finished creating. 191 | deferred.reject(err); 192 | } 193 | }, () => { 194 | ourTrans.internal_markTransactionClosed(); 195 | this._lockHelper!!!.transactionComplete(transToken); 196 | }); 197 | return deferred.promise; 198 | }); 199 | } 200 | } 201 | 202 | class CordovaNativeSqliteTransaction extends SqliteSqlTransaction { 203 | constructor(trans: CordovaTransaction, 204 | protected _lockHelper: TransactionLockHelper, 205 | protected _transToken: TransactionToken, 206 | schema: DbSchema, 207 | verbose: boolean, 208 | maxVariables: number, 209 | supportsFTS3: boolean) { 210 | super(trans, schema, verbose, maxVariables, supportsFTS3); 211 | } 212 | 213 | getCompletionPromise(): Promise { 214 | return this._transToken.completionPromise; 215 | } 216 | 217 | abort(): void { 218 | // This will wrap through to the transaction error path above. 219 | (this._trans as CordovaTransaction).abort('Manually Aborted'); 220 | } 221 | 222 | getErrorHandlerReturnValue(): boolean { 223 | // react-native-sqlite-storage throws on anything but false 224 | return false; 225 | } 226 | 227 | protected _requiresUnicodeReplacement(): boolean { 228 | // TODO dadere (#333863): Possibly limit this to just iOS, since Android seems to handle it properly 229 | return true; 230 | } 231 | } 232 | -------------------------------------------------------------------------------- /src/FullTextSearchHelpers.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * FullTextSearchHelpers.ts 3 | * Author: David de Regt 4 | * Copyright: Microsoft 2017 5 | * 6 | * Reusable helper classes and functions for supporting Full Text Search. 7 | */ 8 | 9 | import { 10 | words, map, mapKeys, deburr, filter, attempt, keyBy, 11 | isError, pickBy, values, every, Dictionary, assign, take, trim as lodashTrim 12 | } from 'lodash'; 13 | import { Ranges, trim } from 'regexp-i18n'; 14 | 15 | import { DbIndex, IndexSchema, QuerySortOrder, FullTextTermResolution, ItemType, KeyType } from './NoSqlProvider'; 16 | import { getValueForSingleKeypath, getSerializedKeyForKeypath } from './NoSqlProviderUtils'; 17 | 18 | const _whitespaceRegexMatch = /\S+/g; 19 | 20 | // Range which excludes all numbers and digits 21 | const stripSpecialRange = Ranges.LETTERS_DIGITS_AND_DIACRITICS.invert(); 22 | 23 | function sqlCompat(value: string): string { 24 | return trim(value, stripSpecialRange); 25 | } 26 | 27 | export function breakAndNormalizeSearchPhrase(phrase: string): string[] { 28 | // Deburr and tolower before using words since words breaks on CaseChanges. 29 | const deconstructedWords = words(deburr(phrase).toLowerCase(), _whitespaceRegexMatch); 30 | // map(mapKeys is faster than uniq since it's just a pile of strings. 31 | const uniqueWordas = map(mapKeys(deconstructedWords), (value, key) => sqlCompat(key)); 32 | return filter(uniqueWordas, word => !!lodashTrim(word)); 33 | } 34 | 35 | export function getFullTextIndexWordsForItem(keyPath: string, item: any): string[] { 36 | const rawString = getValueForSingleKeypath(item, keyPath); 37 | 38 | return breakAndNormalizeSearchPhrase(rawString); 39 | } 40 | 41 | export abstract class DbIndexFTSFromRangeQueries implements DbIndex { 42 | protected _keyPath: string | string[]; 43 | 44 | constructor(protected _indexSchema: IndexSchema|undefined, protected _primaryKeyPath: string | string[]) { 45 | this._keyPath = this._indexSchema ? this._indexSchema.keyPath : this._primaryKeyPath; 46 | } 47 | 48 | fullTextSearch(searchPhrase: string, 49 | resolution: FullTextTermResolution = FullTextTermResolution.And, limit?: number) 50 | : Promise { 51 | if (!this._indexSchema || !this._indexSchema.fullText) { 52 | return Promise.reject('fullTextSearch performed against non-fullText index!'); 53 | } 54 | 55 | const terms = breakAndNormalizeSearchPhrase(searchPhrase); 56 | if (terms.length === 0) { 57 | return Promise.resolve([]); 58 | } 59 | 60 | const promises = map(terms, term => { 61 | const upperEnd = term.substr(0, term.length - 1) + String.fromCharCode(term.charCodeAt(term.length - 1) + 1); 62 | return this.getRange(term, upperEnd, false, true, false, limit); 63 | }); 64 | return Promise.all(promises).then(results => { 65 | const uniquers = attempt(() => { 66 | return map(results, resultSet => keyBy(resultSet, item => 67 | getSerializedKeyForKeypath(item, this._primaryKeyPath))); 68 | }); 69 | if (isError(uniquers)) { 70 | return Promise.reject(uniquers); 71 | } 72 | 73 | if (resolution === FullTextTermResolution.Or) { 74 | const data = values(assign({}, ...uniquers)); 75 | if (limit) { 76 | return take(data, limit); 77 | } 78 | return data; 79 | } 80 | 81 | if (resolution === FullTextTermResolution.And) { 82 | const [first, ...others] = uniquers; 83 | const dic = pickBy(first, (value, key) => every(others, set => key in set)) as Dictionary; 84 | const data = values(dic); 85 | if (limit) { 86 | return take(data, limit); 87 | } 88 | return data; 89 | } 90 | 91 | return Promise.reject('Undefined full text term resolution type'); 92 | }); 93 | } 94 | 95 | abstract getAll(reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number) 96 | : Promise; 97 | abstract getOnly(key: KeyType, reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number) 98 | : Promise; 99 | abstract getRange(keyLowRange: KeyType, keyHighRange: KeyType, lowRangeExclusive?: boolean, highRangeExclusive?: boolean, 100 | reverseOrSortOrder?: boolean|QuerySortOrder, limit?: number, offset?: number): Promise; 101 | abstract countAll(): Promise; 102 | abstract countOnly(key: KeyType): Promise; 103 | abstract countRange(keyLowRange: KeyType, keyHighRange: KeyType, lowRangeExclusive?: boolean, highRangeExclusive?: boolean) 104 | : Promise; 105 | } 106 | -------------------------------------------------------------------------------- /src/InMemoryProvider.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * InMemoryProvider.ts 3 | * Author: David de Regt 4 | * Copyright: Microsoft 2015 5 | * 6 | * NoSqlProvider provider setup for a non-persisted in-memory database backing provider. 7 | */ 8 | import 'core-js/features/set-immediate'; 9 | import { attempt, isError, each, includes, compact, map, find, Dictionary, flatten, assign, filter, keys, reverse } from 'lodash'; 10 | 11 | import { DbIndexFTSFromRangeQueries, getFullTextIndexWordsForItem } from './FullTextSearchHelpers'; 12 | import { 13 | StoreSchema, DbProvider, DbSchema, DbTransaction, 14 | DbIndex, IndexSchema, DbStore, QuerySortOrder, ItemType, KeyPathType, KeyType 15 | } from './NoSqlProvider'; 16 | import { 17 | arrayify, serializeKeyToString, formListOfSerializedKeys, 18 | getSerializedKeyForKeypath, getValueForSingleKeypath 19 | } from './NoSqlProviderUtils'; 20 | import { TransactionLockHelper, TransactionToken } from './TransactionLockHelper'; 21 | 22 | export interface StoreData { 23 | data: Dictionary; 24 | schema: StoreSchema; 25 | } 26 | 27 | let asyncCallbacks: Set<() => void> = new Set(); 28 | 29 | /** 30 | * This function will defer callback of the specified callback lambda until the next JS tick, simulating standard A+ promise behavior 31 | */ 32 | function asyncCallback(callback: () => void): void { 33 | asyncCallbacks.add(callback); 34 | 35 | if (asyncCallbacks.size === 1) { 36 | setImmediate(resolveAsyncCallbacks); 37 | } 38 | } 39 | 40 | function abortAsyncCallback(callback: () => void): void { 41 | asyncCallbacks.delete(callback); 42 | } 43 | 44 | function resolveAsyncCallbacks(): void { 45 | const savedCallbacks = asyncCallbacks; 46 | asyncCallbacks = new Set(); 47 | savedCallbacks.forEach((item) => item()); 48 | } 49 | 50 | // Very simple in-memory dbprovider for handling IE inprivate windows (and unit tests, maybe?) 51 | export class InMemoryProvider extends DbProvider { 52 | private _stores: { [storeName: string]: StoreData } = {}; 53 | 54 | private _lockHelper: TransactionLockHelper | undefined; 55 | 56 | open(dbName: string, schema: DbSchema, wipeIfExists: boolean, verbose: boolean): Promise { 57 | super.open(dbName, schema, wipeIfExists, verbose); 58 | 59 | each(this._schema!!!.stores, storeSchema => { 60 | this._stores[storeSchema.name] = { schema: storeSchema, data: {} }; 61 | }); 62 | 63 | this._lockHelper = new TransactionLockHelper(schema, true); 64 | 65 | return Promise.resolve(undefined); 66 | } 67 | 68 | protected _deleteDatabaseInternal() { 69 | return Promise.resolve(); 70 | } 71 | 72 | openTransaction(storeNames: string[], writeNeeded: boolean): Promise { 73 | return this._lockHelper!!!.openTransaction(storeNames, writeNeeded).then(token => 74 | new InMemoryTransaction(this, this._lockHelper!!!, token)); 75 | } 76 | 77 | close(): Promise { 78 | return this._lockHelper!!!.closeWhenPossible().then(() => { 79 | this._stores = {}; 80 | }); 81 | } 82 | 83 | internal_getStore(name: string): StoreData { 84 | return this._stores[name]; 85 | } 86 | } 87 | 88 | // Notes: Doesn't limit the stores it can fetch to those in the stores it was "created" with, nor does it handle read-only transactions 89 | class InMemoryTransaction implements DbTransaction { 90 | private _stores: Dictionary = {}; 91 | private _transactionCallback?: () => void; 92 | constructor( 93 | private _prov: InMemoryProvider, 94 | private _lockHelper: TransactionLockHelper, 95 | private _transToken: TransactionToken) { 96 | this._transactionCallback = () => { 97 | this._commitTransaction(); 98 | this._lockHelper.transactionComplete(this._transToken); 99 | this._transactionCallback = undefined; 100 | }; 101 | // Close the transaction on the next tick. By definition, anything is completed synchronously here, so after an event tick 102 | // goes by, there can't have been anything pending. 103 | asyncCallback(this._transactionCallback); 104 | } 105 | 106 | private _commitTransaction(): void { 107 | each(this._stores, store => { 108 | store.internal_commitPendingData(); 109 | }); 110 | } 111 | 112 | getCompletionPromise(): Promise { 113 | return this._transToken.completionPromise; 114 | } 115 | 116 | abort(): void { 117 | each(this._stores, store => { 118 | store.internal_rollbackPendingData(); 119 | }); 120 | this._stores = {}; 121 | 122 | if (this._transactionCallback) { 123 | abortAsyncCallback(this._transactionCallback); 124 | this._transactionCallback = undefined; 125 | } 126 | 127 | this._lockHelper.transactionFailed(this._transToken, 'InMemoryTransaction Aborted'); 128 | } 129 | 130 | markCompleted(): void { 131 | // noop 132 | } 133 | 134 | getStore(storeName: string): DbStore { 135 | if (!includes(arrayify(this._transToken.storeNames), storeName)) { 136 | throw new Error('Store not found in transaction-scoped store list: ' + storeName); 137 | } 138 | if (this._stores[storeName]) { 139 | return this._stores[storeName]; 140 | } 141 | const store = this._prov.internal_getStore(storeName); 142 | if (!store) { 143 | throw new Error('Store not found: ' + storeName); 144 | } 145 | const ims = new InMemoryStore(this, store); 146 | this._stores[storeName] = ims; 147 | return ims; 148 | } 149 | 150 | internal_isOpen() { 151 | return !!this._transactionCallback; 152 | } 153 | } 154 | 155 | class InMemoryStore implements DbStore { 156 | private _pendingCommitDataChanges: Dictionary | undefined; 157 | 158 | private _committedStoreData: Dictionary; 159 | private _mergedData: Dictionary; 160 | private _storeSchema: StoreSchema; 161 | 162 | constructor(private _trans: InMemoryTransaction, storeInfo: StoreData) { 163 | this._storeSchema = storeInfo.schema; 164 | this._committedStoreData = storeInfo.data; 165 | 166 | this._mergedData = this._committedStoreData; 167 | } 168 | 169 | private _checkDataClone(): void { 170 | if (!this._pendingCommitDataChanges) { 171 | this._pendingCommitDataChanges = {}; 172 | this._mergedData = assign({}, this._committedStoreData); 173 | } 174 | } 175 | 176 | internal_commitPendingData(): void { 177 | each(this._pendingCommitDataChanges, (val, key) => { 178 | if (val === undefined) { 179 | delete this._committedStoreData[key]; 180 | } else { 181 | this._committedStoreData[key] = val; 182 | } 183 | }); 184 | 185 | this._pendingCommitDataChanges = undefined; 186 | this._mergedData = this._committedStoreData; 187 | } 188 | 189 | internal_rollbackPendingData(): void { 190 | this._pendingCommitDataChanges = undefined; 191 | this._mergedData = this._committedStoreData; 192 | } 193 | 194 | get(key: KeyType): Promise { 195 | if (!this._trans.internal_isOpen()) { 196 | return Promise.reject('InMemoryTransaction already closed'); 197 | } 198 | 199 | const joinedKey = attempt(() => { 200 | return serializeKeyToString(key, this._storeSchema.primaryKeyPath); 201 | }); 202 | if (isError(joinedKey)) { 203 | return Promise.reject(joinedKey); 204 | } 205 | 206 | return Promise.resolve(this._mergedData[joinedKey]); 207 | } 208 | 209 | getMultiple(keyOrKeys: KeyType | KeyType[]): Promise { 210 | if (!this._trans.internal_isOpen()) { 211 | return Promise.reject('InMemoryTransaction already closed'); 212 | } 213 | 214 | const joinedKeys = attempt(() => { 215 | return formListOfSerializedKeys(keyOrKeys, this._storeSchema.primaryKeyPath); 216 | }); 217 | if (isError(joinedKeys)) { 218 | return Promise.reject(joinedKeys); 219 | } 220 | 221 | return Promise.resolve(compact(map(joinedKeys, key => this._mergedData[key]))); 222 | } 223 | 224 | put(itemOrItems: ItemType | ItemType[]): Promise { 225 | if (!this._trans.internal_isOpen()) { 226 | return Promise.reject('InMemoryTransaction already closed'); 227 | } 228 | this._checkDataClone(); 229 | const err = attempt(() => { 230 | for (const item of arrayify(itemOrItems)) { 231 | let pk = getSerializedKeyForKeypath(item, this._storeSchema.primaryKeyPath)!!!; 232 | 233 | this._pendingCommitDataChanges!!![pk] = item; 234 | this._mergedData[pk] = item; 235 | } 236 | }); 237 | if (err) { 238 | return Promise.reject(err); 239 | } 240 | return Promise.resolve(undefined); 241 | } 242 | 243 | remove(keyOrKeys: KeyType | KeyType[]): Promise { 244 | if (!this._trans.internal_isOpen()) { 245 | return Promise.reject('InMemoryTransaction already closed'); 246 | } 247 | this._checkDataClone(); 248 | 249 | const joinedKeys = attempt(() => { 250 | return formListOfSerializedKeys(keyOrKeys, this._storeSchema.primaryKeyPath); 251 | }); 252 | if (isError(joinedKeys)) { 253 | return Promise.reject(joinedKeys); 254 | } 255 | 256 | for (const key of joinedKeys) { 257 | this._pendingCommitDataChanges!!![key] = undefined; 258 | delete this._mergedData[key]; 259 | } 260 | return Promise.resolve(undefined); 261 | } 262 | 263 | openPrimaryKey(): DbIndex { 264 | this._checkDataClone(); 265 | return new InMemoryIndex(this._trans, this._mergedData, undefined, this._storeSchema.primaryKeyPath); 266 | } 267 | 268 | openIndex(indexName: string): DbIndex { 269 | let indexSchema = find(this._storeSchema.indexes, idx => idx.name === indexName); 270 | if (!indexSchema) { 271 | throw new Error('Index not found: ' + indexName); 272 | } 273 | 274 | this._checkDataClone(); 275 | return new InMemoryIndex(this._trans, this._mergedData, indexSchema, this._storeSchema.primaryKeyPath); 276 | } 277 | 278 | clearAllData(): Promise { 279 | if (!this._trans.internal_isOpen()) { 280 | return Promise.reject('InMemoryTransaction already closed'); 281 | } 282 | this._checkDataClone(); 283 | each(this._mergedData, (val, key) => { 284 | this._pendingCommitDataChanges!!![key] = undefined; 285 | }); 286 | this._mergedData = {}; 287 | return Promise.resolve(undefined); 288 | } 289 | } 290 | 291 | // Note: Currently maintains nothing interesting -- rebuilds the results every time from scratch. Scales like crap. 292 | class InMemoryIndex extends DbIndexFTSFromRangeQueries { 293 | constructor(private _trans: InMemoryTransaction, private _mergedData: Dictionary, 294 | indexSchema: IndexSchema | undefined, primaryKeyPath: KeyPathType) { 295 | super(indexSchema, primaryKeyPath); 296 | } 297 | 298 | // Warning: This function can throw, make sure to trap. 299 | private _calcChunkedData(): Dictionary | Dictionary { 300 | if (!this._indexSchema) { 301 | // Primary key -- use data intact 302 | return this._mergedData; 303 | } 304 | 305 | // If it's not the PK index, re-pivot the data to be keyed off the key value built from the keypath 306 | let data: Dictionary = {}; 307 | each(this._mergedData, item => { 308 | // Each item may be non-unique so store as an array of items for each key 309 | let keys: string[]; 310 | if (this._indexSchema!!!.fullText) { 311 | keys = map(getFullTextIndexWordsForItem(this._keyPath, item), val => 312 | serializeKeyToString(val, this._keyPath)); 313 | } else if (this._indexSchema!!!.multiEntry) { 314 | // Have to extract the multiple entries into this alternate table... 315 | const valsRaw = getValueForSingleKeypath(item, this._keyPath); 316 | if (valsRaw) { 317 | keys = map(arrayify(valsRaw), val => 318 | serializeKeyToString(val, this._keyPath)); 319 | } else { 320 | keys = []; 321 | } 322 | } else { 323 | keys = [getSerializedKeyForKeypath(item, this._keyPath)!!!]; 324 | } 325 | 326 | for (const key of keys) { 327 | if (!data[key]) { 328 | data[key] = [item]; 329 | } else { 330 | data[key].push(item); 331 | } 332 | } 333 | }); 334 | return data; 335 | } 336 | 337 | getAll(reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number): Promise { 338 | if (!this._trans.internal_isOpen()) { 339 | return Promise.reject('InMemoryTransaction already closed'); 340 | } 341 | 342 | const data = attempt(() => { 343 | return this._calcChunkedData(); 344 | }); 345 | if (isError(data)) { 346 | return Promise.reject(data); 347 | } 348 | 349 | const sortedKeys = keys(data).sort(); 350 | return this._returnResultsFromKeys(data, sortedKeys, reverseOrSortOrder, limit, offset); 351 | } 352 | 353 | getOnly(key: KeyType, reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number) 354 | : Promise { 355 | return this.getRange(key, key, false, false, reverseOrSortOrder, limit, offset); 356 | } 357 | 358 | getRange(keyLowRange: KeyType, keyHighRange: KeyType, lowRangeExclusive?: boolean, highRangeExclusive?: boolean, 359 | reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number): Promise { 360 | if (!this._trans.internal_isOpen()) { 361 | return Promise.reject('InMemoryTransaction already closed'); 362 | } 363 | 364 | let data: Dictionary | Dictionary; 365 | let sortedKeys: string[]; 366 | const err = attempt(() => { 367 | data = this._calcChunkedData(); 368 | sortedKeys = this._getKeysForRange(data, keyLowRange, keyHighRange, lowRangeExclusive, highRangeExclusive).sort(); 369 | }); 370 | if (err) { 371 | return Promise.reject(err); 372 | } 373 | 374 | return this._returnResultsFromKeys(data!!!, sortedKeys!!!, reverseOrSortOrder, limit, offset); 375 | } 376 | 377 | // Warning: This function can throw, make sure to trap. 378 | private _getKeysForRange(data: Dictionary | Dictionary, keyLowRange: KeyType, keyHighRange: KeyType, 379 | lowRangeExclusive?: boolean, highRangeExclusive?: boolean): string[] { 380 | const keyLow = serializeKeyToString(keyLowRange, this._keyPath); 381 | const keyHigh = serializeKeyToString(keyHighRange, this._keyPath); 382 | return filter(keys(data), key => 383 | (key > keyLow || (key === keyLow && !lowRangeExclusive)) && (key < keyHigh || (key === keyHigh && !highRangeExclusive))); 384 | } 385 | 386 | private _returnResultsFromKeys(data: Dictionary | Dictionary, sortedKeys: string[], 387 | reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number) { 388 | if (reverseOrSortOrder === true || reverseOrSortOrder === QuerySortOrder.Reverse) { 389 | sortedKeys = reverse(sortedKeys); 390 | } 391 | 392 | if (offset) { 393 | sortedKeys = sortedKeys.slice(offset); 394 | } 395 | 396 | if (limit) { 397 | sortedKeys = sortedKeys.slice(0, limit); 398 | } 399 | 400 | let results = map(sortedKeys, key => data[key]); 401 | return Promise.resolve(flatten(results)); 402 | } 403 | 404 | countAll(): Promise { 405 | if (!this._trans.internal_isOpen()) { 406 | return Promise.reject('InMemoryTransaction already closed'); 407 | } 408 | const data = attempt(() => { 409 | return this._calcChunkedData(); 410 | }); 411 | if (isError(data)) { 412 | return Promise.reject(data); 413 | } 414 | return Promise.resolve(keys(data).length); 415 | } 416 | 417 | countOnly(key: KeyType): Promise { 418 | return this.countRange(key, key, false, false); 419 | } 420 | 421 | countRange(keyLowRange: KeyType, keyHighRange: KeyType, lowRangeExclusive?: boolean, highRangeExclusive?: boolean) 422 | : Promise { 423 | if (!this._trans.internal_isOpen()) { 424 | return Promise.reject('InMemoryTransaction already closed'); 425 | } 426 | 427 | const keys = attempt(() => { 428 | const data = this._calcChunkedData(); 429 | return this._getKeysForRange(data, keyLowRange, keyHighRange, lowRangeExclusive, highRangeExclusive); 430 | }); 431 | if (isError(keys)) { 432 | return Promise.reject(keys); 433 | } 434 | 435 | return Promise.resolve(keys.length); 436 | } 437 | } 438 | -------------------------------------------------------------------------------- /src/IndexedDbProvider.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * IndexedDbProvider.ts 3 | * Author: David de Regt 4 | * Copyright: Microsoft 2015 5 | * 6 | * NoSqlProvider provider setup for IndexedDB, a web browser storage module. 7 | */ 8 | 9 | import { each, some, find, includes, isObject, attempt, isError, map, filter, compact, clone, isArray, noop, isUndefined } from 'lodash'; 10 | import { getWindow } from './get-window'; 11 | import { DbIndexFTSFromRangeQueries, getFullTextIndexWordsForItem } from './FullTextSearchHelpers'; 12 | import { 13 | DbProvider, DbSchema, DbStore, DbTransaction, StoreSchema, DbIndex, 14 | QuerySortOrder, IndexSchema, ItemType, KeyPathType, KeyType 15 | } from './NoSqlProvider'; 16 | import { 17 | isIE, isCompoundKeyPath, serializeKeyToString, getKeyForKeypath, arrayify, formListOfKeys, 18 | getValueForSingleKeypath, getSerializedKeyForKeypath 19 | } from './NoSqlProviderUtils'; 20 | import { TransactionLockHelper, TransactionToken } from './TransactionLockHelper'; 21 | import { defer } from './defer'; 22 | const IndexPrefix = 'nsp_i_'; 23 | 24 | // Extending interfaces that should be in lib.d.ts but aren't for some reason. 25 | declare global { 26 | interface Window { 27 | _indexedDB: IDBFactory; 28 | mozIndexedDB: IDBFactory; 29 | webkitIndexedDB: IDBFactory; 30 | msIndexedDB: IDBFactory; 31 | } 32 | } 33 | 34 | function getBrowserInfo() { 35 | // From https://stackoverflow.com/questions/5916900/how-can-you-detect-the-version-of-a-browser 36 | let ua = navigator.userAgent, tem, M = ua.match(/(opera|chrome|safari|firefox|msie|trident(?=\/))\/?\s*(\d+)/i) || []; 37 | if (/trident/i.test(M[1])) { 38 | tem = /\brv[ :]+(\d+)/g.exec(ua) || []; 39 | return { name: 'IE', version: (tem[1] || '') }; 40 | } 41 | if (M[1] === 'Chrome') { 42 | tem = ua.match(/\bOPR|Edge\/(\d+)/); 43 | if (tem != null) { 44 | return { name: 'Opera', version: tem[1] }; 45 | } 46 | } 47 | M = M[2] ? [M[1], M[2]] : [navigator.appName, navigator.appVersion, '-?']; 48 | if ((tem = ua.match(/version\/(\d+)/i)) != null) { 49 | M.splice(1, 1, tem[1]); 50 | } 51 | return { 52 | name: M[0], 53 | version: M[1], 54 | }; 55 | } 56 | 57 | // The DbProvider implementation for IndexedDB. This one is fairly straightforward since the library's access patterns pretty 58 | // closely mirror IndexedDB's. We mostly do a lot of wrapping of the APIs into JQuery promises and have some fancy footwork to 59 | // do semi-automatic schema upgrades. 60 | export class IndexedDbProvider extends DbProvider { 61 | private _db: IDBDatabase | undefined; 62 | private _dbFactory: IDBFactory; 63 | private _fakeComplicatedKeys: boolean; 64 | 65 | private _lockHelper: TransactionLockHelper | undefined; 66 | 67 | // By default, it uses the in-browser indexed db factory, but you can pass in an explicit factory. Currently only used for unit tests. 68 | constructor(explicitDbFactory?: IDBFactory, explicitDbFactorySupportsCompoundKeys?: boolean) { 69 | super(); 70 | 71 | if (explicitDbFactory) { 72 | this._dbFactory = explicitDbFactory; 73 | this._fakeComplicatedKeys = !explicitDbFactorySupportsCompoundKeys; 74 | } else { 75 | const win = getWindow(); 76 | this._dbFactory = win._indexedDB || win.indexedDB || win.mozIndexedDB || win.webkitIndexedDB || win.msIndexedDB; 77 | 78 | if (typeof explicitDbFactorySupportsCompoundKeys !== 'undefined') { 79 | this._fakeComplicatedKeys = !explicitDbFactorySupportsCompoundKeys; 80 | } else { 81 | // IE/Edge's IndexedDB implementation doesn't support compound keys, so we have to fake it by implementing them similar to 82 | // how the WebSqlProvider does, by concatenating the values into another field which then gets its own index. 83 | this._fakeComplicatedKeys = isIE(); 84 | } 85 | } 86 | } 87 | 88 | static WrapRequest(req: IDBRequest): Promise { 89 | const task = defer(); 90 | 91 | req.onsuccess = (/*ev*/) => { 92 | task.resolve(req.result); 93 | }; 94 | req.onerror = (ev) => { 95 | task.reject(ev); 96 | }; 97 | 98 | return task.promise; 99 | } 100 | 101 | open(dbName: string, schema: DbSchema, wipeIfExists: boolean, verbose: boolean): Promise { 102 | // Note: DbProvider returns null instead of a promise that needs waiting for. 103 | super.open(dbName, schema, wipeIfExists, verbose); 104 | 105 | if (!this._dbFactory) { 106 | // Couldn't even find a supported indexeddb object on the browser... 107 | return Promise.reject('No support for IndexedDB in this browser'); 108 | } 109 | 110 | if (typeof (navigator) !== 'undefined') { 111 | // In a browser of some sort, so check for some known deficient IndexedDB implementations... 112 | 113 | const browserInfo = getBrowserInfo(); 114 | if (browserInfo.name === 'Safari' && Number(browserInfo.version) < 10) { 115 | // Safari < 10 doesn't support indexeddb properly, so don't let it try 116 | return Promise.reject('Safari versions before 10.0 don\'t properly implement IndexedDB'); 117 | } 118 | 119 | if (navigator.userAgent.indexOf('Mobile Crosswalk') !== -1) { 120 | // Android crosswalk indexeddb is slow, don't use it 121 | return Promise.reject('Android Crosswalk\'s IndexedDB implementation is very slow'); 122 | } 123 | } 124 | 125 | if (wipeIfExists) { 126 | try { 127 | this._dbFactory.deleteDatabase(dbName); 128 | } catch (e) { 129 | // Don't care 130 | } 131 | } 132 | 133 | this._lockHelper = new TransactionLockHelper(schema, true); 134 | 135 | const dbOpen = this._dbFactory.open(dbName, schema.version); 136 | 137 | let migrationPutters: Promise[] = []; 138 | 139 | dbOpen.onupgradeneeded = (event) => { 140 | const db: IDBDatabase = dbOpen.result; 141 | const target = (event.currentTarget || event.target); 142 | const trans = target.transaction; 143 | 144 | if (!trans) { 145 | throw new Error('onupgradeneeded: target is null!'); 146 | } 147 | 148 | if (schema.lastUsableVersion && event.oldVersion < schema.lastUsableVersion) { 149 | // Clear all stores if it's past the usable version 150 | console.log('Old version detected (' + event.oldVersion + '), clearing all data'); 151 | each(db.objectStoreNames, name => { 152 | db.deleteObjectStore(name); 153 | }); 154 | } 155 | 156 | // Delete dead stores 157 | each(db.objectStoreNames, storeName => { 158 | if (!some(schema.stores, store => store.name === storeName)) { 159 | db.deleteObjectStore(storeName); 160 | } 161 | }); 162 | 163 | // Create all stores 164 | for (const storeSchema of schema.stores) { 165 | let store: IDBObjectStore; 166 | const storeExistedBefore = includes(db.objectStoreNames, storeSchema.name); 167 | if (!storeExistedBefore) { // store doesn't exist yet 168 | let primaryKeyPath = storeSchema.primaryKeyPath; 169 | if (this._fakeComplicatedKeys && isCompoundKeyPath(primaryKeyPath)) { 170 | // Going to have to hack the compound primary key index into a column, so here it is. 171 | primaryKeyPath = 'nsp_pk'; 172 | } 173 | 174 | // Any is to fix a lib.d.ts issue in TS 2.0.3 - it doesn't realize that keypaths can be compound for some reason... 175 | store = db.createObjectStore(storeSchema.name, { keyPath: primaryKeyPath } as any); 176 | } else { // store exists, might need to update indexes and migrate the data 177 | store = trans.objectStore(storeSchema.name); 178 | 179 | // Check for any indexes no longer in the schema or have been changed 180 | each(store.indexNames, indexName => { 181 | const index = store.index(indexName); 182 | 183 | let nuke = false; 184 | const indexSchema = find(storeSchema.indexes, idx => idx.name === indexName); 185 | if (!indexSchema || !isObject(indexSchema)) { 186 | nuke = true; 187 | } else if (typeof index.keyPath !== typeof indexSchema.keyPath) { 188 | nuke = true; 189 | } else if (typeof index.keyPath === 'string') { 190 | if (index.keyPath !== indexSchema.keyPath) { 191 | nuke = true; 192 | } 193 | } else /* Keypath is array */ if (index.keyPath.length !== indexSchema.keyPath.length) { 194 | // Keypath length doesn't match, don't bother doing a comparison of each element 195 | nuke = true; 196 | } else { 197 | for (let i = 0; i < index.keyPath.length; i++) { 198 | if (index.keyPath[i] !== indexSchema.keyPath[i]) { 199 | nuke = true; 200 | break; 201 | } 202 | } 203 | } 204 | 205 | if (nuke) { 206 | store.deleteIndex(indexName); 207 | } 208 | }); 209 | } 210 | 211 | // IndexedDB deals well with adding new indexes on the fly, so we don't need to force migrate, 212 | // unless adding multiEntry or fullText index 213 | let needsMigrate = false; 214 | // Check any indexes in the schema that need to be created 215 | if (storeSchema.indexes) { 216 | for (const indexSchema of storeSchema.indexes) { 217 | if (!includes(store.indexNames, indexSchema.name)) { 218 | const keyPath = indexSchema.keyPath; 219 | if (this._fakeComplicatedKeys) { 220 | if (indexSchema.multiEntry || indexSchema.fullText) { 221 | if (isCompoundKeyPath(keyPath)) { 222 | throw new Error('Can\'t use multiEntry and compound keys'); 223 | } else { 224 | // Create an object store for the index 225 | let indexStore = db.createObjectStore(storeSchema.name + '_' + indexSchema.name, 226 | { autoIncrement: true }); 227 | indexStore.createIndex('key', 'key'); 228 | indexStore.createIndex('refkey', 'refkey'); 229 | 230 | if (storeExistedBefore && !indexSchema.doNotBackfill) { 231 | needsMigrate = true; 232 | } 233 | } 234 | } else if (isCompoundKeyPath(keyPath)) { 235 | // Going to have to hack the compound index into a column, so here it is. 236 | store.createIndex(indexSchema.name, IndexPrefix + indexSchema.name, { 237 | unique: indexSchema.unique 238 | }); 239 | } else { 240 | store.createIndex(indexSchema.name, keyPath, { 241 | unique: indexSchema.unique 242 | }); 243 | } 244 | } else if (indexSchema.fullText) { 245 | store.createIndex(indexSchema.name, IndexPrefix + indexSchema.name, { 246 | unique: false, 247 | multiEntry: true 248 | }); 249 | 250 | if (storeExistedBefore && !indexSchema.doNotBackfill) { 251 | needsMigrate = true; 252 | } 253 | } else { 254 | store.createIndex(indexSchema.name, keyPath, { 255 | unique: indexSchema.unique, 256 | multiEntry: indexSchema.multiEntry 257 | }); 258 | } 259 | } 260 | } 261 | } 262 | 263 | if (needsMigrate) { 264 | // Walk every element in the store and re-put it to fill out the new index. 265 | const fakeToken: TransactionToken = { 266 | storeNames: [storeSchema.name], 267 | exclusive: false, 268 | completionPromise: defer().promise 269 | }; 270 | const iTrans = new IndexedDbTransaction(trans, undefined, fakeToken, schema, this._fakeComplicatedKeys); 271 | const tStore = iTrans.getStore(storeSchema.name); 272 | 273 | const cursorReq = store.openCursor(); 274 | let thisIndexPutters: Promise[] = []; 275 | migrationPutters.push(IndexedDbIndex.iterateOverCursorRequest(cursorReq, cursor => { 276 | const err = attempt(() => { 277 | const item = removeFullTextMetadataAndReturn(storeSchema, (cursor as any).value); 278 | 279 | thisIndexPutters.push(tStore.put(item)); 280 | }); 281 | if (err) { 282 | thisIndexPutters.push(Promise.reject(err)); 283 | } 284 | }).then(() => Promise.all(thisIndexPutters).then(noop))); 285 | } 286 | } 287 | }; 288 | 289 | const promise = IndexedDbProvider.WrapRequest(dbOpen); 290 | 291 | return promise.then(db => { 292 | return Promise.all(migrationPutters).then(() => { 293 | this._db = db; 294 | }); 295 | }, err => { 296 | if (err && err.type === 'error' && err.target && err.target.error && err.target.error.name === 'VersionError') { 297 | if (!wipeIfExists) { 298 | console.log('Database version too new, Wiping: ' + (err.target.error.message || err.target.error.name)); 299 | 300 | return this.open(dbName, schema, true, verbose); 301 | } 302 | } 303 | return Promise.reject(err); 304 | }); 305 | } 306 | 307 | close(): Promise { 308 | if (!this._db) { 309 | return Promise.reject('Database already closed'); 310 | } 311 | 312 | this._db.close(); 313 | this._db = undefined; 314 | return Promise.resolve(undefined); 315 | } 316 | 317 | protected _deleteDatabaseInternal(): Promise { 318 | const trans = attempt(() => { 319 | return this._dbFactory.deleteDatabase(this._dbName!!!); 320 | }); 321 | 322 | if (isError(trans)) { 323 | return Promise.reject(trans); 324 | } 325 | 326 | const deferred = defer(); 327 | 328 | trans.onsuccess = () => { 329 | deferred.resolve(void 0); 330 | }; 331 | trans.onerror = (ev) => { 332 | deferred.reject(ev); 333 | }; 334 | 335 | return deferred.promise; 336 | } 337 | 338 | openTransaction(storeNames: string[], writeNeeded: boolean): Promise { 339 | if (!this._db) { 340 | return Promise.reject('Can\'t openTransaction, database is closed'); 341 | } 342 | 343 | let intStoreNames = storeNames; 344 | 345 | if (this._fakeComplicatedKeys) { 346 | // Clone the list becuase we're going to add fake store names to it 347 | intStoreNames = clone(storeNames); 348 | 349 | // Pull the alternate multientry stores into the transaction as well 350 | let missingStores: string[] = []; 351 | for (const storeName of storeNames) { 352 | let storeSchema = find(this._schema!!!.stores, s => s.name === storeName); 353 | if (!storeSchema) { 354 | missingStores.push(storeName); 355 | continue; 356 | } 357 | if (storeSchema.indexes) { 358 | for (const indexSchema of storeSchema.indexes) { 359 | if (indexSchema.multiEntry || indexSchema.fullText) { 360 | intStoreNames.push(storeSchema!!!.name + '_' + indexSchema.name); 361 | } 362 | } 363 | } 364 | } 365 | if (missingStores.length > 0) { 366 | return Promise.reject('Can\'t find store(s): ' + missingStores.join(',')); 367 | } 368 | } 369 | 370 | return this._lockHelper!!!.openTransaction(storeNames, writeNeeded).then(transToken => { 371 | const trans = attempt(() => { 372 | return this._db!!!.transaction(intStoreNames, writeNeeded ? 'readwrite' : 'readonly'); 373 | }); 374 | if (isError(trans)) { 375 | return Promise.reject(trans); 376 | } 377 | 378 | return new IndexedDbTransaction(trans, this._lockHelper, transToken, this._schema!!!, this._fakeComplicatedKeys); 379 | }); 380 | } 381 | } 382 | 383 | // DbTransaction implementation for the IndexedDB DbProvider. 384 | class IndexedDbTransaction implements DbTransaction { 385 | private _stores: IDBObjectStore[]; 386 | 387 | constructor(private _trans: IDBTransaction, lockHelper: TransactionLockHelper | undefined, private _transToken: TransactionToken, 388 | private _schema: DbSchema, private _fakeComplicatedKeys: boolean) { 389 | this._stores = map(this._transToken.storeNames, storeName => this._trans.objectStore(storeName)); 390 | 391 | if (lockHelper) { 392 | // Chromium seems to have a bug in their indexeddb implementation that lets it start a timeout 393 | // while the app is in the middle of a commit (it does a two-phase commit). It can then finish 394 | // the commit, and later fire the timeout, despite the transaction having been written out already. 395 | // In this case, it appears that we should be completely fine to ignore the spurious timeout. 396 | // 397 | // Applicable Chromium source code here: 398 | // https://chromium.googlesource.com/chromium/src/+/master/content/browser/indexed_db/indexed_db_transaction.cc 399 | let history: string[] = []; 400 | 401 | this._trans.oncomplete = () => { 402 | history.push('complete'); 403 | 404 | lockHelper.transactionComplete(this._transToken); 405 | }; 406 | 407 | this._trans.onerror = () => { 408 | history.push('error-' + (this._trans.error ? this._trans.error.message : '')); 409 | 410 | if (history.length > 1) { 411 | console.warn('IndexedDbTransaction Errored after Resolution, Swallowing. Error: ' + 412 | (this._trans.error ? this._trans.error.message : undefined) + ', History: ' + history.join(',')); 413 | return; 414 | } 415 | 416 | lockHelper.transactionFailed(this._transToken, 'IndexedDbTransaction OnError: ' + 417 | (this._trans.error ? this._trans.error.message : undefined) + ', History: ' + history.join(',')); 418 | }; 419 | 420 | this._trans.onabort = () => { 421 | history.push('abort-' + (this._trans.error ? this._trans.error.message : '')); 422 | 423 | if (history.length > 1) { 424 | console.warn('IndexedDbTransaction Aborted after Resolution, Swallowing. Error: ' + 425 | (this._trans.error ? this._trans.error.message : undefined) + ', History: ' + history.join(',')); 426 | return; 427 | } 428 | 429 | lockHelper.transactionFailed(this._transToken, 'IndexedDbTransaction Aborted, Error: ' + 430 | (this._trans.error ? this._trans.error.message : undefined) + ', History: ' + history.join(',')); 431 | }; 432 | } 433 | } 434 | 435 | getStore(storeName: string): DbStore { 436 | const store = find(this._stores, s => s.name === storeName); 437 | const storeSchema = find(this._schema.stores, s => s.name === storeName); 438 | if (!store || !storeSchema) { 439 | throw new Error('Store not found: ' + storeName); 440 | } 441 | 442 | const indexStores: IDBObjectStore[] = []; 443 | if (this._fakeComplicatedKeys && storeSchema.indexes) { 444 | // Pull the alternate multientry stores in as well 445 | for (const indexSchema of storeSchema.indexes) { 446 | if (indexSchema.multiEntry || indexSchema.fullText) { 447 | indexStores.push(this._trans.objectStore(storeSchema.name + '_' + indexSchema.name)); 448 | } 449 | } 450 | } 451 | 452 | return new IndexedDbStore(store, indexStores, storeSchema, this._fakeComplicatedKeys); 453 | } 454 | 455 | getCompletionPromise(): Promise { 456 | return this._transToken.completionPromise; 457 | } 458 | 459 | abort(): void { 460 | // This will wrap through the onAbort above 461 | this._trans.abort(); 462 | } 463 | 464 | markCompleted(): void { 465 | // noop 466 | } 467 | } 468 | 469 | function removeFullTextMetadataAndReturn(schema: StoreSchema, val: T): T { 470 | if (val && schema.indexes) { 471 | // We have full text index fields as real fields on the result, so nuke them before returning them to the caller. 472 | for (const index of schema.indexes) { 473 | if (index.fullText) { 474 | delete (val as any)[IndexPrefix + index.name]; 475 | } 476 | } 477 | } 478 | 479 | return val; 480 | } 481 | 482 | // DbStore implementation for the IndexedDB DbProvider. Again, fairly closely maps to the standard IndexedDB spec, aside from 483 | // a bunch of hacks to support compound keypaths on IE. 484 | class IndexedDbStore implements DbStore { 485 | constructor(private _store: IDBObjectStore, private _indexStores: IDBObjectStore[], private _schema: StoreSchema, 486 | private _fakeComplicatedKeys: boolean) { 487 | // NOP 488 | } 489 | 490 | get(key: KeyType): Promise { 491 | if (this._fakeComplicatedKeys && isCompoundKeyPath(this._schema.primaryKeyPath)) { 492 | const err = attempt(() => { 493 | key = serializeKeyToString(key, this._schema.primaryKeyPath); 494 | }); 495 | if (err) { 496 | return Promise.reject(err); 497 | } 498 | } 499 | 500 | return IndexedDbProvider.WrapRequest(this._store.get(key)) 501 | .then(val => removeFullTextMetadataAndReturn(this._schema, val)); 502 | } 503 | 504 | getMultiple(keyOrKeys: KeyType | KeyType[]): Promise { 505 | const keys = attempt(() => { 506 | const keys = formListOfKeys(keyOrKeys, this._schema.primaryKeyPath); 507 | 508 | if (this._fakeComplicatedKeys && isCompoundKeyPath(this._schema.primaryKeyPath)) { 509 | return map(keys, key => serializeKeyToString(key, this._schema.primaryKeyPath)); 510 | } 511 | return keys; 512 | }); 513 | if (isError(keys)) { 514 | return Promise.reject(keys); 515 | } 516 | 517 | // There isn't a more optimized way to do this with indexeddb, have to get the results one by one 518 | return Promise.all( 519 | map(keys, key => IndexedDbProvider.WrapRequest(this._store.get(key)) 520 | .then(val => removeFullTextMetadataAndReturn(this._schema, val)))) 521 | .then(compact); 522 | } 523 | 524 | put(itemOrItems: ItemType | ItemType[]): Promise { 525 | let items = arrayify(itemOrItems); 526 | 527 | let promises: Promise[] = []; 528 | 529 | const err = attempt(() => { 530 | for (const item of items) { 531 | let errToReport: any; 532 | let fakedPk = false; 533 | 534 | if (this._fakeComplicatedKeys) { 535 | // Fill out any compound-key indexes 536 | if (isCompoundKeyPath(this._schema.primaryKeyPath)) { 537 | fakedPk = true; 538 | (item as any)['nsp_pk'] = getSerializedKeyForKeypath(item, this._schema.primaryKeyPath); 539 | } 540 | 541 | if (this._schema.indexes) { 542 | for (const index of this._schema.indexes) { 543 | if (index.multiEntry || index.fullText) { 544 | let indexStore = find(this._indexStores, store => store.name === this._schema.name + '_' + index.name)!!!; 545 | 546 | let keys: any[]; 547 | if (index.fullText) { 548 | keys = getFullTextIndexWordsForItem(index.keyPath, item); 549 | } else { 550 | // Get each value of the multientry and put it into the index store 551 | const valsRaw = getValueForSingleKeypath(item, index.keyPath); 552 | // It might be an array of multiple entries, so just always go with array-based logic 553 | keys = arrayify(valsRaw); 554 | } 555 | 556 | let refKey: any; 557 | const err = attempt(() => { 558 | // We're using normal indexeddb tables to store the multientry indexes, so we only need to use the key 559 | // serialization if the multientry keys ALSO are compound. 560 | if (isCompoundKeyPath(index.keyPath)) { 561 | keys = map(keys, val => serializeKeyToString(val, index.keyPath)); 562 | } 563 | 564 | // We need to reference the PK of the actual row we're using here, so calculate the actual PK -- if 565 | // it's compound, we're already faking complicated keys, so we know to serialize it to a string. If 566 | // not, use the raw value. 567 | refKey = getKeyForKeypath(item, this._schema.primaryKeyPath); 568 | if (isArray(this._schema.primaryKeyPath)) { 569 | refKey = serializeKeyToString(refKey, this._schema.primaryKeyPath); 570 | } 571 | }); 572 | 573 | if (err) { 574 | errToReport = err; 575 | break; 576 | } 577 | 578 | // First clear out the old values from the index store for the refkey 579 | const cursorReq = indexStore.index('refkey').openCursor(IDBKeyRange.only(refKey)); 580 | promises.push(IndexedDbIndex.iterateOverCursorRequest(cursorReq, cursor => { 581 | cursor['delete'](); 582 | }) 583 | .then(() => { 584 | // After nuking the existing entries, add the new ones 585 | let iputters = map(keys, key => { 586 | const indexObj = { 587 | key: key, 588 | refkey: refKey 589 | }; 590 | return IndexedDbProvider.WrapRequest(indexStore.put(indexObj)); 591 | }); 592 | return Promise.all(iputters); 593 | }).then(noop)); 594 | } else if (isCompoundKeyPath(index.keyPath)) { 595 | (item as any)[IndexPrefix + index.name] = 596 | getSerializedKeyForKeypath(item, index.keyPath); 597 | } 598 | } 599 | } 600 | } else if (this._schema.indexes) { 601 | for (const index of this._schema.indexes) { 602 | if (index.fullText) { 603 | (item as any)[IndexPrefix + index.name] = 604 | getFullTextIndexWordsForItem(index.keyPath, item); 605 | } 606 | } 607 | } 608 | 609 | if (!errToReport) { 610 | errToReport = attempt(() => { 611 | const req = this._store.put(item); 612 | 613 | if (fakedPk) { 614 | // If we faked the PK and mutated the incoming object, we can nuke that on the way out. IndexedDB clones the 615 | // object synchronously for the put call, so it's already been captured with the nsp_pk field intact. 616 | delete (item as any)['nsp_pk']; 617 | } 618 | 619 | promises.push(IndexedDbProvider.WrapRequest(req)); 620 | }); 621 | } 622 | 623 | if (errToReport) { 624 | promises.push(Promise.reject(errToReport)); 625 | } 626 | } 627 | }); 628 | 629 | if (err) { 630 | return Promise.reject(err); 631 | } 632 | 633 | return Promise.all(promises).then(noop); 634 | } 635 | 636 | remove(keyOrKeys: KeyType | KeyType[]): Promise { 637 | const keys = attempt(() => { 638 | const keys = formListOfKeys(keyOrKeys, this._schema.primaryKeyPath); 639 | 640 | if (this._fakeComplicatedKeys && isCompoundKeyPath(this._schema.primaryKeyPath)) { 641 | return map(keys, key => serializeKeyToString(key, this._schema.primaryKeyPath)); 642 | } 643 | return keys; 644 | }); 645 | if (isError(keys)) { 646 | return Promise.reject(keys); 647 | } 648 | 649 | return Promise.all(map(keys, key => { 650 | if (this._fakeComplicatedKeys && some(this._schema.indexes, index => index.multiEntry || index.fullText)) { 651 | // If we're faking keys and there's any multientry indexes, we have to do the way more complicated version... 652 | return IndexedDbProvider.WrapRequest(this._store.get(key)).then(item => { 653 | if (item) { 654 | // Go through each multiEntry index and nuke the referenced items from the sub-stores 655 | let promises = map(filter(this._schema.indexes, index => !!index.multiEntry), index => { 656 | let indexStore = find(this._indexStores, store => store.name === this._schema.name + '_' + index.name)!!!; 657 | const refKey = attempt(() => { 658 | // We need to reference the PK of the actual row we're using here, so calculate the actual PK -- if it's 659 | // compound, we're already faking complicated keys, so we know to serialize it to a string. If not, use the 660 | // raw value. 661 | const tempRefKey = getKeyForKeypath(item, this._schema.primaryKeyPath)!!!; 662 | return isArray(this._schema.primaryKeyPath) ? 663 | serializeKeyToString(tempRefKey, this._schema.primaryKeyPath) : 664 | tempRefKey; 665 | }); 666 | if (isError(refKey)) { 667 | return Promise.reject(refKey); 668 | } 669 | 670 | // First clear out the old values from the index store for the refkey 671 | const cursorReq = indexStore.index('refkey').openCursor(IDBKeyRange.only(refKey)); 672 | return IndexedDbIndex.iterateOverCursorRequest(cursorReq, cursor => { 673 | cursor['delete'](); 674 | }); 675 | }); 676 | // Also remember to nuke the item from the actual store 677 | promises.push(IndexedDbProvider.WrapRequest(this._store['delete'](key))); 678 | return Promise.all(promises).then(noop); 679 | } 680 | return undefined; 681 | }); 682 | } 683 | 684 | return IndexedDbProvider.WrapRequest(this._store['delete'](key)); 685 | })).then(noop); 686 | } 687 | 688 | openIndex(indexName: string): DbIndex { 689 | const indexSchema = find(this._schema.indexes, idx => idx.name === indexName); 690 | if (!indexSchema) { 691 | throw new Error('Index not found: ' + indexName); 692 | } 693 | 694 | if (this._fakeComplicatedKeys && (indexSchema.multiEntry || indexSchema.fullText)) { 695 | const store = find(this._indexStores, indexStore => indexStore.name === this._schema.name + '_' + indexSchema.name); 696 | if (!store) { 697 | throw new Error('Indexstore not found: ' + this._schema.name + '_' + indexSchema.name); 698 | } 699 | return new IndexedDbIndex(store.index('key'), indexSchema, this._schema.primaryKeyPath, this._fakeComplicatedKeys, 700 | this._store); 701 | } else { 702 | const index = this._store.index(indexName); 703 | if (!index) { 704 | throw new Error('Index store not found: ' + indexName); 705 | } 706 | return new IndexedDbIndex(index, indexSchema, this._schema.primaryKeyPath, this._fakeComplicatedKeys); 707 | } 708 | } 709 | 710 | openPrimaryKey(): DbIndex { 711 | return new IndexedDbIndex(this._store, undefined, this._schema.primaryKeyPath, this._fakeComplicatedKeys); 712 | } 713 | 714 | clearAllData(): Promise { 715 | let storesToClear = [this._store]; 716 | if (this._indexStores) { 717 | storesToClear = storesToClear.concat(this._indexStores); 718 | } 719 | 720 | let promises = map(storesToClear, store => IndexedDbProvider.WrapRequest(store.clear())); 721 | 722 | return Promise.all(promises).then(noop); 723 | } 724 | } 725 | 726 | // DbIndex implementation for the IndexedDB DbProvider. Fairly closely maps to the standard IndexedDB spec, aside from 727 | // a bunch of hacks to support compound keypaths on IE and some helpers to make the caller not have to walk the awkward cursor 728 | // result APIs to get their result list. Also added ability to use an "index" for opening the primary key on a store. 729 | class IndexedDbIndex extends DbIndexFTSFromRangeQueries { 730 | constructor(private _store: IDBIndex | IDBObjectStore, indexSchema: IndexSchema | undefined, 731 | primaryKeyPath: KeyPathType, private _fakeComplicatedKeys: boolean, private _fakedOriginalStore?: IDBObjectStore) { 732 | super(indexSchema, primaryKeyPath); 733 | } 734 | 735 | private _resolveCursorResult(req: IDBRequest, limit?: number, offset?: number): Promise { 736 | if (this._fakeComplicatedKeys && this._fakedOriginalStore) { 737 | // Get based on the keys from the index store, which have refkeys that point back to the original store 738 | return IndexedDbIndex.getFromCursorRequest<{ key: string, refkey: any }>(req, limit, offset).then(rets => { 739 | // Now get the original items using the refkeys from the index store, which are PKs on the main store 740 | const getters = map(rets, ret => IndexedDbProvider.WrapRequest<{ key: string, refkey: any }>( 741 | this._fakedOriginalStore!!!.get(ret.refkey))); 742 | return Promise.all(getters); 743 | }); 744 | } else { 745 | return IndexedDbIndex.getFromCursorRequest(req, limit, offset); 746 | } 747 | } 748 | 749 | private _isGetAllApiAvailable(reverse?: number, offset?: number): boolean { 750 | return !reverse && !offset && !isUndefined(this._store.getAll) && !this._fakeComplicatedKeys; 751 | } 752 | 753 | getAll(reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number): Promise { 754 | const reverse = reverseOrSortOrder === true || reverseOrSortOrder === QuerySortOrder.Reverse; 755 | if (this._isGetAllApiAvailable(limit, offset)) { 756 | return IndexedDbProvider.WrapRequest(this._store.getAll(undefined, limit)); 757 | } 758 | // ************************* Don't change this null to undefined, IE chokes on it... ***************************** 759 | // ************************* Don't change this null to undefined, IE chokes on it... ***************************** 760 | // ************************* Don't change this null to undefined, IE chokes on it... ***************************** 761 | const req = this._store.openCursor(null!!!, reverse ? 'prev' : 'next'); 762 | return this._resolveCursorResult(req, limit, offset); 763 | } 764 | 765 | getOnly(key: KeyType, reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number) 766 | : Promise { 767 | const keyRange = attempt(() => { 768 | return this._getKeyRangeForOnly(key); 769 | }); 770 | if (isError(keyRange)) { 771 | return Promise.reject(keyRange); 772 | } 773 | const reverse = reverseOrSortOrder === true || reverseOrSortOrder === QuerySortOrder.Reverse; 774 | if (this._isGetAllApiAvailable(limit, offset)) { 775 | return IndexedDbProvider.WrapRequest(this._store.getAll(keyRange, limit)); 776 | } 777 | const req = this._store.openCursor(keyRange, reverse ? 'prev' : 'next'); 778 | return this._resolveCursorResult(req, limit, offset); 779 | } 780 | 781 | // Warning: This function can throw, make sure to trap. 782 | private _getKeyRangeForOnly(key: KeyType): IDBKeyRange { 783 | if (this._fakeComplicatedKeys && isCompoundKeyPath(this._keyPath)) { 784 | return IDBKeyRange.only(serializeKeyToString(key, this._keyPath)); 785 | } 786 | return IDBKeyRange.only(key); 787 | } 788 | 789 | getRange(keyLowRange: KeyType, keyHighRange: KeyType, lowRangeExclusive?: boolean, highRangeExclusive?: boolean, 790 | reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number): Promise { 791 | const keyRange = attempt(() => { 792 | return this._getKeyRangeForRange(keyLowRange, keyHighRange, lowRangeExclusive, highRangeExclusive); 793 | }); 794 | if (isError(keyRange)) { 795 | return Promise.reject(keyRange); 796 | } 797 | 798 | const reverse = reverseOrSortOrder === true || reverseOrSortOrder === QuerySortOrder.Reverse; 799 | if (this._isGetAllApiAvailable(limit, offset)) { 800 | return IndexedDbProvider.WrapRequest(this._store.getAll(keyRange, limit)); 801 | } 802 | const req = this._store.openCursor(keyRange, reverse ? 'prev' : 'next'); 803 | return this._resolveCursorResult(req, limit, offset); 804 | } 805 | 806 | // Warning: This function can throw, make sure to trap. 807 | private _getKeyRangeForRange(keyLowRange: KeyType, keyHighRange: KeyType, lowRangeExclusive?: boolean, 808 | highRangeExclusive?: boolean) 809 | : IDBKeyRange { 810 | if (this._fakeComplicatedKeys && isCompoundKeyPath(this._keyPath)) { 811 | // IE has to switch to hacky pre-joined-compound-keys 812 | return IDBKeyRange.bound(serializeKeyToString(keyLowRange, this._keyPath), 813 | serializeKeyToString(keyHighRange, this._keyPath), 814 | lowRangeExclusive, highRangeExclusive); 815 | } 816 | return IDBKeyRange.bound(keyLowRange, keyHighRange, lowRangeExclusive, highRangeExclusive); 817 | } 818 | 819 | countAll(): Promise { 820 | const req = this._store.count(); 821 | return this._countRequest(req); 822 | } 823 | 824 | countOnly(key: KeyType): Promise { 825 | const keyRange = attempt(() => { 826 | return this._getKeyRangeForOnly(key); 827 | }); 828 | if (isError(keyRange)) { 829 | return Promise.reject(keyRange); 830 | } 831 | 832 | const req = this._store.count(keyRange); 833 | return this._countRequest(req); 834 | } 835 | 836 | countRange(keyLowRange: KeyType, keyHighRange: KeyType, lowRangeExclusive?: boolean, highRangeExclusive?: boolean) 837 | : Promise { 838 | let keyRange = attempt(() => { 839 | return this._getKeyRangeForRange(keyLowRange, keyHighRange, lowRangeExclusive, highRangeExclusive); 840 | }); 841 | if (isError(keyRange)) { 842 | return Promise.reject(keyRange); 843 | } 844 | 845 | const req = this._store.count(keyRange); 846 | return this._countRequest(req); 847 | } 848 | 849 | static getFromCursorRequest(req: IDBRequest, limit?: number, offset?: number): Promise { 850 | let outList: T[] = []; 851 | return this.iterateOverCursorRequest(req, cursor => { 852 | // Typings on cursor are wrong... 853 | outList.push((cursor as any).value); 854 | }, limit, offset).then(() => { 855 | return outList; 856 | }); 857 | } 858 | 859 | private _countRequest(req: IDBRequest): Promise { 860 | const deferred = defer(); 861 | 862 | req.onsuccess = (event) => { 863 | deferred.resolve((event.target).result as number); 864 | }; 865 | req.onerror = (ev) => { 866 | deferred.reject(ev); 867 | }; 868 | 869 | return deferred.promise; 870 | } 871 | 872 | static iterateOverCursorRequest(req: IDBRequest, func: (cursor: IDBCursor) => void, limit?: number, offset?: number) 873 | : Promise { 874 | const deferred = defer(); 875 | 876 | let count = 0; 877 | req.onsuccess = (event) => { 878 | const cursor: IDBCursor = (event.target).result; 879 | if (cursor) { 880 | if (offset) { 881 | cursor.advance(offset); 882 | offset = 0; 883 | } else { 884 | func(cursor); 885 | count++; 886 | if (limit && (count === limit)) { 887 | deferred.resolve(void 0); 888 | return; 889 | } 890 | cursor['continue'](); 891 | } 892 | } else { 893 | // Nothing else to iterate 894 | deferred.resolve(void 0); 895 | } 896 | }; 897 | req.onerror = (ev) => { 898 | deferred.reject(ev); 899 | }; 900 | 901 | return deferred.promise; 902 | } 903 | } 904 | -------------------------------------------------------------------------------- /src/NoSqlProvider.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * NoSqlProvider.ts 3 | * Author: David de Regt 4 | * Copyright: Microsoft 2016 5 | * 6 | * Low-level wrapper to expose a nosql-like database which can be backed by 7 | * numerous different backend store types, invisible to the consumer. The 8 | * usage semantics are very similar to IndexedDB. This file contains most 9 | * of the helper interfaces, while the specific database providers should 10 | * be required piecemeal. 11 | */ 12 | import './Promise.extensions'; // See the file for why it is being imported like this. 13 | import { noop, attempt, isError } from 'lodash'; 14 | import { defer } from './defer'; 15 | 16 | // Basic nomenclature types for everyone to agree on. 17 | export type ItemType = object; 18 | export type KeyComponentType = string|number|Date; 19 | export type KeyType = KeyComponentType|KeyComponentType[]; 20 | export type KeyPathType = string | string[]; 21 | export enum QuerySortOrder { 22 | None, 23 | Forward, 24 | Reverse 25 | } 26 | 27 | // Schema type describing an index for a store. 28 | export interface IndexSchema { 29 | name: string; 30 | keyPath: KeyPathType; 31 | unique?: boolean; 32 | multiEntry?: boolean; 33 | fullText?: boolean; 34 | includeDataInIndex?: boolean; 35 | doNotBackfill?: boolean; 36 | } 37 | 38 | // Schema type describing a data store. Must give a keypath for the primary key for the store. Further indexes are optional. 39 | export interface StoreSchema { 40 | name: string; 41 | indexes?: IndexSchema[]; 42 | primaryKeyPath: KeyPathType; 43 | // Estimated object size to enable batched data migration. Default = 200 44 | estimatedObjBytes?: number; 45 | } 46 | 47 | // Schema representing a whole database (a collection of stores). Change your version number whenever you change your schema or 48 | // the new schema will have no effect, as it only checks schema differences during a version change process. 49 | export interface DbSchema { 50 | version: number; 51 | // If set, during the upgrade path, all versions below this will be cleared and built from scratch rather than upgraded 52 | lastUsableVersion?: number; 53 | stores: StoreSchema[]; 54 | } 55 | 56 | export enum FullTextTermResolution { 57 | And, 58 | Or 59 | } 60 | 61 | // Interface type describing an index being opened for querying. 62 | export interface DbIndex { 63 | getAll(reverseOrSortOrder?: boolean|QuerySortOrder, limit?: number, offset?: number): Promise; 64 | getOnly(key: KeyType, reverseOrSortOrder?: boolean|QuerySortOrder, limit?: number, offset?: number): Promise; 65 | getRange(keyLowRange: KeyType, keyHighRange: KeyType, lowRangeExclusive?: boolean, highRangeExclusive?: boolean, 66 | reverseOrSortOrder?: boolean|QuerySortOrder, limit?: number, offset?: number): Promise; 67 | countAll(): Promise; 68 | countOnly(key: KeyType): Promise; 69 | countRange(keyLowRange: KeyType, keyHighRange: KeyType, lowRangeExclusive?: boolean, highRangeExclusive?: boolean) 70 | : Promise; 71 | fullTextSearch(searchPhrase: string, resolution?: FullTextTermResolution, limit?: number): Promise; 72 | } 73 | 74 | // Interface type describing a database store opened for accessing. Get commands at this level work against the primary keypath 75 | // of the store. 76 | export interface DbStore { 77 | get(key: KeyType): Promise; 78 | getMultiple(keyOrKeys: KeyType|KeyType[]): Promise; 79 | put(itemOrItems: ItemType|ItemType[]): Promise; 80 | remove(keyOrKeys: KeyType|KeyType[]): Promise; 81 | 82 | openPrimaryKey(): DbIndex; 83 | openIndex(indexName: string): DbIndex; 84 | 85 | clearAllData(): Promise; 86 | } 87 | 88 | // Interface type describing a transaction. All accesses to a database must go through a transaction, though the provider has 89 | // shortcut accessor functions that get a transaction for you for the one-off queries. 90 | export interface DbTransaction { 91 | getStore(storeName: string): DbStore; 92 | // This promise will resolve when the transaction commits, or will reject when there's a transaction-level error. 93 | getCompletionPromise(): Promise; 94 | // Attempt to abort the transaction (if it hasn't yet completed or aborted). Completion will be detectable via the 95 | // getCompletionPromise() promise. 96 | abort(): void; 97 | // This method is noop for most of implementations 98 | // react-native implementation could use this as an opportunity to finish transactions without additional bridge delay 99 | markCompleted(): void; 100 | } 101 | 102 | // Abstract base type for a database provider. Has accessors for opening transactions and one-off accesor helpers. 103 | // Note: this is a different concept than a DbStore or DbIndex, although it provides a similar (or identical) interface. 104 | export abstract class DbProvider { 105 | protected _dbName: string|undefined; 106 | protected _schema: DbSchema|undefined; 107 | protected _verbose: boolean|undefined; 108 | 109 | open(dbName: string, schema: DbSchema, wipeIfExists: boolean, verbose: boolean): Promise { 110 | // virtual call 111 | this._dbName = dbName; 112 | this._schema = schema; 113 | this._verbose = verbose; 114 | return undefined!!!; 115 | } 116 | 117 | abstract close(): Promise; 118 | 119 | // You must perform all of your actions on the transaction handed to you in the callback block without letting it expire. 120 | // When the last callback from the last executed action against the DbTransaction is executed, the transaction closes, so be very 121 | // careful using deferrals/promises that may wait for the main thread to close out before handling your response. 122 | // Undefined for storeNames means ALL stores. 123 | abstract openTransaction(storeNames: string[]|undefined, writeNeeded: boolean): Promise; 124 | 125 | deleteDatabase(): Promise { 126 | return this.close().always(() => this._deleteDatabaseInternal()); 127 | } 128 | 129 | clearAllData(): Promise { 130 | var storeNames = this._schema!!!.stores.map(store => store.name); 131 | 132 | return this.openTransaction(storeNames, true).then(trans => { 133 | const clearers = storeNames.map(storeName => { 134 | const store = attempt(() => { 135 | return trans.getStore(storeName); 136 | }); 137 | if (!store || isError(store)) { 138 | return Promise.reject('Store "' + storeName + '" not found'); 139 | } 140 | return store.clearAllData(); 141 | }); 142 | return Promise.all(clearers).then(noop); 143 | }); 144 | } 145 | 146 | protected abstract _deleteDatabaseInternal(): Promise; 147 | 148 | private _getStoreTransaction(storeName: string, readWrite: boolean): Promise { 149 | return this.openTransaction([storeName], readWrite).then(trans => { 150 | const store = attempt(() => { 151 | return trans.getStore(storeName); 152 | }); 153 | if (!store || isError(store)) { 154 | return Promise.reject('Store "' + storeName + '" not found'); 155 | } 156 | return store; 157 | }); 158 | } 159 | 160 | // Shortcut functions 161 | get(storeName: string, key: KeyType): Promise { 162 | return this._getStoreTransaction(storeName, false).then(store => { 163 | return store.get(key); 164 | }); 165 | } 166 | 167 | getMultiple(storeName: string, keyOrKeys: KeyType|KeyType[]): Promise { 168 | return this._getStoreTransaction(storeName, false).then(store => { 169 | return store.getMultiple(keyOrKeys); 170 | }); 171 | } 172 | 173 | put(storeName: string, itemOrItems: ItemType|ItemType[]): Promise { 174 | return this._getStoreTransaction(storeName, true).then(store => { 175 | return store.put(itemOrItems); 176 | }); 177 | } 178 | 179 | remove(storeName: string, keyOrKeys: KeyType|KeyType[]): Promise { 180 | return this._getStoreTransaction(storeName, true).then(store => { 181 | return store.remove(keyOrKeys); 182 | }); 183 | } 184 | 185 | private _getStoreIndexTransaction(storeName: string, readWrite: boolean, indexName: string|undefined): Promise { 186 | return this._getStoreTransaction(storeName, readWrite).then(store => { 187 | const index = attempt(() => { 188 | return indexName ? store.openIndex(indexName) : store.openPrimaryKey(); 189 | }); 190 | if (!index || isError(index)) { 191 | return Promise.reject('Index "' + indexName + '" not found'); 192 | } 193 | return index; 194 | }); 195 | } 196 | 197 | getAll(storeName: string, indexName: string|undefined, reverseOrSortOrder?: boolean|QuerySortOrder, limit?: number, offset?: number) 198 | : Promise { 199 | return this._getStoreIndexTransaction(storeName, false, indexName).then(index => { 200 | return index.getAll(reverseOrSortOrder, limit, offset); 201 | }); 202 | } 203 | 204 | getOnly(storeName: string, indexName: string | undefined, key: KeyType, reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, 205 | offset?: number): Promise { 206 | return this._getStoreIndexTransaction(storeName, false, indexName).then(index => { 207 | return index.getOnly(key, reverseOrSortOrder, limit, offset); 208 | }); 209 | } 210 | 211 | getRange(storeName: string, indexName: string | undefined, keyLowRange: KeyType, keyHighRange: KeyType, lowRangeExclusive?: boolean, 212 | highRangeExclusive?: boolean, reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number) 213 | : Promise { 214 | return this._getStoreIndexTransaction(storeName, false, indexName).then(index => { 215 | return index.getRange(keyLowRange, keyHighRange, lowRangeExclusive, highRangeExclusive, reverseOrSortOrder, limit, offset); 216 | }); 217 | } 218 | 219 | countAll(storeName: string, indexName: string|undefined): Promise { 220 | return this._getStoreIndexTransaction(storeName, false, indexName).then(index => { 221 | return index.countAll(); 222 | }); 223 | } 224 | 225 | countOnly(storeName: string, indexName: string|undefined, key: KeyType): Promise { 226 | return this._getStoreIndexTransaction(storeName, false, indexName).then(index => { 227 | return index.countOnly(key); 228 | }); 229 | } 230 | 231 | countRange(storeName: string, indexName: string|undefined, keyLowRange: KeyType, keyHighRange: KeyType, 232 | lowRangeExclusive?: boolean, highRangeExclusive?: boolean): Promise { 233 | return this._getStoreIndexTransaction(storeName, false, indexName).then(index => { 234 | return index.countRange(keyLowRange, keyHighRange, lowRangeExclusive, highRangeExclusive); 235 | }); 236 | } 237 | 238 | fullTextSearch(storeName: string, indexName: string, searchPhrase: string, 239 | resolution: FullTextTermResolution = FullTextTermResolution.And, limit?: number): Promise { 240 | return this._getStoreIndexTransaction(storeName, false, indexName).then(index => { 241 | return index.fullTextSearch(searchPhrase, resolution); 242 | }); 243 | } 244 | } 245 | 246 | // Runs down the given providers in order and tries to instantiate them. If they're not supported, it will continue until it finds one 247 | // that does work, or it will reject the promise if it runs out of providers and none work. 248 | export function openListOfProviders(providersToTry: DbProvider[], dbName: string, schema: DbSchema, wipeIfExists: boolean = false, 249 | verbose: boolean = false): Promise { 250 | const task = defer(); 251 | let providerIndex = 0; 252 | let errorList: any[] = []; 253 | 254 | var tryNext = () => { 255 | if (providerIndex >= providersToTry.length) { 256 | task.reject(errorList.length <= 1 ? errorList[0] : errorList); 257 | return; 258 | } 259 | 260 | var provider = providersToTry[providerIndex]; 261 | provider.open(dbName, schema, wipeIfExists, verbose).then(() => { 262 | task.resolve(provider); 263 | }, (err) => { 264 | errorList.push(err); 265 | providerIndex++; 266 | tryNext(); 267 | }); 268 | }; 269 | 270 | tryNext(); 271 | 272 | return task.promise; 273 | } 274 | -------------------------------------------------------------------------------- /src/NoSqlProviderUtils.ts: -------------------------------------------------------------------------------- 1 |  /** 2 | * NoSqlProviderUtils.ts 3 | * Author: David de Regt 4 | * Copyright: Microsoft 2015 5 | * 6 | * Reusable helper functions for NoSqlProvider providers/transactions/etc. 7 | */ 8 | 9 | import { map, isArray, some, isNull, isDate, get, isUndefined, isObject } from 'lodash'; 10 | 11 | import { KeyComponentType, KeyPathType, KeyType } from './NoSqlProvider'; 12 | 13 | // Extending interfaces that should be in lib.d.ts but aren't for some reason. 14 | declare global { 15 | interface Document { 16 | documentMode: number; 17 | } 18 | } 19 | 20 | export function isIE() { 21 | return (typeof (document) !== 'undefined' && document.all !== null && document.documentMode <= 11) || 22 | (typeof (navigator) !== 'undefined' && !!navigator.userAgent && navigator.userAgent.indexOf('Edge/') !== -1); 23 | } 24 | 25 | export function isSafari() { 26 | return (typeof (navigator) !== 'undefined' && ((navigator.userAgent.indexOf('Safari') !== -1 && 27 | navigator.userAgent.indexOf('Chrome') === -1 && navigator.userAgent.indexOf('BB10') === -1) || 28 | (navigator.userAgent.indexOf('Mobile Crosswalk') !== -1))); 29 | } 30 | 31 | export function arrayify(obj: T | T[]): T[] { 32 | return isArray(obj) ? obj : [obj]; 33 | } 34 | 35 | // Constant string for joining compound keypaths for websql and IE indexeddb. There may be marginal utility in using a more obscure 36 | // string sequence. 37 | const keypathJoinerString = '%&'; 38 | 39 | // This function computes a serialized single string value for a keypath on an object. This is used for generating ordered string keys 40 | // for compound (or non-compound) values. 41 | export function getSerializedKeyForKeypath(obj: any, keyPathRaw: KeyPathType): string|undefined { 42 | const values = getKeyForKeypath(obj, keyPathRaw); 43 | if (values === undefined) { 44 | return undefined; 45 | } 46 | 47 | return serializeKeyToString(values, keyPathRaw); 48 | } 49 | 50 | export function getKeyForKeypath(obj: any, keyPathRaw: KeyPathType): KeyType|undefined { 51 | const keyPathArray = arrayify(keyPathRaw); 52 | 53 | const values = map(keyPathArray, kp => getValueForSingleKeypath(obj, kp)); 54 | if (some(values, val => isNull(val) || isUndefined(val))) { 55 | // If any components of the key are null/undefined, then the result is undefined 56 | return undefined; 57 | } 58 | 59 | if (!isArray(keyPathRaw)) { 60 | return values[0]; 61 | } else { 62 | return values; 63 | } 64 | } 65 | 66 | // Internal helper function for getting a value out of a standard keypath. 67 | export function getValueForSingleKeypath(obj: any, singleKeyPath: string): any { 68 | return get(obj, singleKeyPath, undefined); 69 | } 70 | 71 | export function isCompoundKeyPath(keyPath: KeyPathType) { 72 | return isArray(keyPath) && keyPath.length > 1; 73 | } 74 | 75 | export function formListOfKeys(keyOrKeys: KeyType|KeyType[], keyPath: KeyPathType): any[] { 76 | if (isCompoundKeyPath(keyPath)) { 77 | if (!isArray(keyOrKeys)) { 78 | throw new Error('formListOfKeys called with a compound keypath (' + JSON.stringify(keyPath) + 79 | ') but a non-compound keyOrKeys (' + JSON.stringify(keyOrKeys) + ')'); 80 | } 81 | if (!isArray(keyOrKeys[0])) { 82 | // Looks like a single compound key, so make it a list of a single key 83 | return [keyOrKeys]; 84 | } 85 | 86 | // Array of arrays, so looks fine 87 | return keyOrKeys; 88 | } 89 | 90 | // Non-compound, so just make sure it's a list when it comes out in case it's a single key passed 91 | return arrayify(keyOrKeys); 92 | } 93 | 94 | export function serializeValueToOrderableString(val: KeyComponentType): string { 95 | if (typeof val === 'number') { 96 | return 'A' + serializeNumberToOrderableString(val as number); 97 | } 98 | if (isDate(val)) { 99 | return 'B' + serializeNumberToOrderableString((val as Date).getTime()); 100 | } 101 | if (typeof val === 'string') { 102 | return 'C' + (val as string); 103 | } 104 | 105 | const type = isObject(val) ? Object.getPrototypeOf(val).constructor : typeof val; 106 | throw new Error('Type \'' + type + '\' unsupported at this time. Only numbers, Dates, and strings are currently supported.'); 107 | } 108 | 109 | const zeroes = '0000000000000000'; 110 | 111 | function formatFixed(n: number, digits: number): string { 112 | var result = String(n); 113 | var prefix = digits - result.length; 114 | 115 | if (prefix > 0 && prefix < zeroes.length) { 116 | result = zeroes.substr(0, prefix) + result; 117 | } 118 | 119 | return result; 120 | } 121 | 122 | export function serializeNumberToOrderableString(n: number) { 123 | if (n === 0 || isNaN(n) || !isFinite(n)) { 124 | return String(n); 125 | } 126 | 127 | var isPositive = true; 128 | 129 | if (n < 0) { 130 | isPositive = false; 131 | n = -n; 132 | } 133 | 134 | var exponent = Math.floor(Math.log(n) / Math.LN10); 135 | 136 | n = n / Math.pow(10, exponent); 137 | 138 | if (isPositive) { 139 | return formatFixed(1024 + exponent, 4) + String(n); 140 | } else { 141 | return '-' + formatFixed(1024 - exponent, 4) + String(10 - n); 142 | } 143 | } 144 | 145 | export function serializeKeyToString(key: KeyType, keyPath: KeyPathType): string { 146 | if (isCompoundKeyPath(keyPath)) { 147 | if (isArray(key)) { 148 | return map(key, k => serializeValueToOrderableString(k)).join(keypathJoinerString); 149 | } else { 150 | throw new Error('serializeKeyToString called with a compound keypath (' + JSON.stringify(keyPath) + 151 | ') but a non-compound key (' + JSON.stringify(key) + ')'); 152 | } 153 | } else { 154 | if (isArray(key)) { 155 | throw new Error('serializeKeyToString called with a non-compound keypath (' + JSON.stringify(keyPath) + 156 | ') but a compound key (' + JSON.stringify(key) + ')'); 157 | } else { 158 | return serializeValueToOrderableString(key); 159 | } 160 | } 161 | } 162 | 163 | export function formListOfSerializedKeys(keyOrKeys: KeyType|KeyType[], keyPath: KeyPathType): string[] { 164 | return map(formListOfKeys(keyOrKeys, keyPath), key => serializeKeyToString(key, keyPath)); 165 | } 166 | -------------------------------------------------------------------------------- /src/NodeSqlite3DbProvider.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * NodeSqlite3DbProvider.ts 3 | * Author: David de Regt 4 | * Copyright: Microsoft 2015 5 | * 6 | * NoSqlProvider provider setup for NodeJs to use a sqlite3-based provider. 7 | * Can pass :memory: to the dbName for it to use an in-memory sqlite instance that's blown away each close() call. 8 | */ 9 | 10 | import { Database, verbose } from 'sqlite3'; 11 | 12 | import { defer } from './defer'; 13 | import { DbSchema } from './NoSqlProvider'; 14 | import { SqlProviderBase, SqlTransaction } from './SqlProviderBase'; 15 | import { TransactionLockHelper, TransactionToken } from './TransactionLockHelper'; 16 | 17 | export default class NodeSqlite3DbProvider extends SqlProviderBase { 18 | private _db: Database|undefined; 19 | 20 | private _lockHelper: TransactionLockHelper|undefined; 21 | 22 | constructor(supportsFTS3 = true) { 23 | super(supportsFTS3); 24 | } 25 | 26 | open(dbName: string, schema: DbSchema, wipeIfExists: boolean, setVerbose: boolean): Promise { 27 | super.open(dbName, schema, wipeIfExists, setVerbose); 28 | 29 | if (setVerbose) { 30 | verbose(); 31 | } 32 | 33 | this._db = new Database(dbName); 34 | 35 | this._lockHelper = new TransactionLockHelper(schema, false); 36 | 37 | return this._ourVersionChecker(wipeIfExists); 38 | } 39 | 40 | openTransaction(storeNames: string[], writeNeeded: boolean): Promise { 41 | if (!this._db) { 42 | return Promise.reject('Can\'t openTransaction on a closed database'); 43 | } 44 | if (this._verbose) { 45 | console.log('openTransaction Called with Stores: ' + (storeNames ? storeNames.join(',') : undefined) + 46 | ', WriteNeeded: ' + writeNeeded); 47 | } 48 | 49 | return this._lockHelper!!!.openTransaction(storeNames, writeNeeded).then(transToken => { 50 | if (this._verbose) { 51 | console.log('openTransaction Resolved with Stores: ' + (storeNames ? storeNames.join(',') : undefined) + 52 | ', WriteNeeded: ' + writeNeeded); 53 | } 54 | const trans = new NodeSqlite3Transaction(this._db!!!, this._lockHelper!!!, transToken, this._schema!!!, this._verbose!!!, 55 | this._supportsFTS3); 56 | if (writeNeeded) { 57 | return trans.runQuery('BEGIN EXCLUSIVE TRANSACTION').then(ret => trans); 58 | } 59 | return trans; 60 | }); 61 | } 62 | 63 | close(): Promise { 64 | if (!this._db) { 65 | return Promise.reject('Database already closed'); 66 | } 67 | return this._lockHelper!!!.closeWhenPossible().then(() => { 68 | let task = defer(); 69 | this._db!!!.close((err) => { 70 | this._db = undefined; 71 | if (err) { 72 | task.reject(err); 73 | } else { 74 | task.resolve(void 0); 75 | } 76 | }); 77 | return task.promise; 78 | }); 79 | } 80 | 81 | protected _deleteDatabaseInternal(): Promise { 82 | return Promise.reject('No support for deleting'); 83 | } 84 | } 85 | 86 | class NodeSqlite3Transaction extends SqlTransaction { 87 | private _openTimer: number|undefined; 88 | private _openQueryCount = 0; 89 | 90 | constructor(private _db: Database, private _lockHelper: TransactionLockHelper, private _transToken: TransactionToken, 91 | schema: DbSchema, verbose: boolean, supportsFTS3: boolean) { 92 | super(schema, verbose, 999, supportsFTS3); 93 | 94 | this._setTimer(); 95 | } 96 | 97 | private _clearTimer(): void { 98 | if (this._openTimer) { 99 | clearTimeout(this._openTimer); 100 | this._openTimer = undefined; 101 | } 102 | } 103 | 104 | private _setTimer(): void { 105 | this._clearTimer(); 106 | this._openTimer = setTimeout(() => { 107 | this._openTimer = undefined; 108 | 109 | if (!this._transToken.exclusive) { 110 | this.internal_markTransactionClosed(); 111 | this._lockHelper.transactionComplete(this._transToken); 112 | return; 113 | } 114 | 115 | this.runQuery('COMMIT TRANSACTION').then(() => { 116 | this._clearTimer(); 117 | this.internal_markTransactionClosed(); 118 | this._lockHelper.transactionComplete(this._transToken); 119 | }); 120 | }, 0) as any as number; 121 | } 122 | 123 | getCompletionPromise(): Promise { 124 | return this._transToken.completionPromise; 125 | } 126 | 127 | abort(): void { 128 | this._clearTimer(); 129 | 130 | if (!this._transToken.exclusive) { 131 | this.internal_markTransactionClosed(); 132 | this._lockHelper.transactionFailed(this._transToken, 'NodeSqlite3Transaction Aborted'); 133 | return; 134 | } 135 | 136 | this.runQuery('ROLLBACK TRANSACTION').always(() => { 137 | this._clearTimer(); 138 | this.internal_markTransactionClosed(); 139 | this._lockHelper.transactionFailed(this._transToken, 'NodeSqlite3Transaction Aborted'); 140 | }); 141 | } 142 | 143 | runQuery(sql: string, parameters: any[] = []): Promise { 144 | if (!this._isTransactionOpen()) { 145 | return Promise.reject('SqliteSqlTransaction already closed'); 146 | } 147 | 148 | this._clearTimer(); 149 | this._openQueryCount++; 150 | 151 | const deferred = defer(); 152 | 153 | if (this._verbose) { 154 | console.log('Query: ' + sql + (parameters ? ', Args: ' + JSON.stringify(parameters) : '')); 155 | } 156 | 157 | var stmt = this._db.prepare(sql); 158 | stmt.bind.apply(stmt, parameters); 159 | stmt.all((err, rows) => { 160 | this._openQueryCount--; 161 | if (this._openQueryCount === 0) { 162 | this._setTimer(); 163 | } 164 | 165 | if (err) { 166 | console.error('Query Error: SQL: ' + sql + ', Error: ' + err.toString()); 167 | deferred.reject(err); 168 | } else { 169 | deferred.resolve(rows); 170 | } 171 | 172 | stmt.finalize(); 173 | }); 174 | 175 | return deferred.promise; 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /src/Promise.d.ts: -------------------------------------------------------------------------------- 1 | declare interface Promise { 2 | always: (func: (value: T | any) => U | PromiseLike) => Promise; 3 | } -------------------------------------------------------------------------------- /src/Promise.extensions.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This file contains a series of promise polyfills that help with making promises 3 | * work on legacy platforms see https://caniuse.com/promises for a full list of 4 | * browsers where this can be used. 5 | * 6 | * NOTE: Promise.finally is added to the ES standard but is only available on more 7 | * recent versions https://caniuse.com/mdn-javascript_builtins_promise_finally 8 | * 9 | * The fila lso adds support for Promise.always (NOT ES compliant). The difference between 10 | * finally and always is, always will always rethrow the exception thrown in the catch(). 11 | * 12 | * How to import this file: 13 | * As this file contains polyfills and added type definitions you would need to import it with side-effects 14 | * import "./Promise.extensions" 15 | * not doing so will prevent the polyfills from being imported. 16 | * 17 | * Additionally, make sure this file is imported as part of the index.ts (entry-point) for your bundle/chunk 18 | * to make sure all Promise prototypes are correctly patched. 19 | * 20 | * NOTE: You do not need to do anything to import this file when importing NoSQLProvider... this file is 21 | * automatically included for NoSQLProvider as part of NoSQLProvider.ts which is the main entry-point for this package 22 | */ 23 | import 'core-js/features/promise'; 24 | import 'core-js/features/promise/finally'; 25 | import { getWindow } from './get-window'; 26 | 27 | /** 28 | * Below is a polyfill for Promise.always 29 | */ 30 | if (!Promise.prototype.always) { 31 | Promise.prototype.always = function (onResolveOrReject) { 32 | return this.then(onResolveOrReject, 33 | function (reason: any) { 34 | onResolveOrReject(reason); 35 | throw reason; 36 | }); 37 | }; 38 | } 39 | 40 | export interface PromiseConfig { 41 | // If we catch exceptions in success/fail blocks, it silently falls back to the fail case of the outer promise. 42 | // If this is global variable is true, it will also spit out a console.error with the exception for debugging. 43 | exceptionsToConsole: boolean; 44 | 45 | // Whether or not to actually attempt to catch exceptions with try/catch blocks inside the resolution cases. 46 | // Disable this for debugging when you'd rather the debugger caught the exception synchronously rather than 47 | // digging through a stack trace. 48 | catchExceptions: boolean; 49 | 50 | // Regardless of whether an exception is caught or not, this will always execute. 51 | exceptionHandler: ((ex: Error) => void) | undefined; 52 | 53 | // If an ErrorFunc is not added to the task (then, catch, always) before the task rejects or synchonously 54 | // after that, then this function is called with the error. Default throws the error. 55 | unhandledErrorHandler: (err: any) => void; 56 | } 57 | 58 | let boundRejectionHandledListener: (e: PromiseRejectionEvent) => void; 59 | let boundUnhandledRejectionListener: (e: PromiseRejectionEvent) => void; 60 | 61 | /** 62 | * This function provides a way to override the default handling behavior for ES6 promises 63 | * @param config various configuration options for this. 64 | */ 65 | export function registerPromiseGlobalHandlers(config: PromiseConfig) { 66 | boundRejectionHandledListener = (e: PromiseRejectionEvent) => { 67 | if (config.exceptionsToConsole) { 68 | console.error('handled', e.reason, e.promise); 69 | } 70 | 71 | if (config.exceptionHandler) { 72 | config.exceptionHandler(e.reason); 73 | } 74 | }; 75 | boundUnhandledRejectionListener = (e: PromiseRejectionEvent) => { 76 | if (config.exceptionsToConsole) { 77 | console.error('unhandled', e.reason, e.promise); 78 | } 79 | 80 | if (config.catchExceptions) { 81 | e.preventDefault(); 82 | } 83 | config.unhandledErrorHandler(e.reason); 84 | }; 85 | getWindow().addEventListener('rejectionhandled', boundRejectionHandledListener); 86 | 87 | getWindow().addEventListener('unhandledrejection', boundUnhandledRejectionListener); 88 | } 89 | 90 | /** 91 | * This function provides a way to unregister global listeners for promises. 92 | * @param config various configuration options for this. 93 | */ 94 | export function unRegisterPromiseGlobalHandlers() { 95 | getWindow().removeEventListener('rejectionhandled', boundRejectionHandledListener); 96 | 97 | getWindow().removeEventListener('unhandledrejection', boundUnhandledRejectionListener); 98 | } 99 | -------------------------------------------------------------------------------- /src/SqlProviderBase.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * SqlProviderBase.ts 3 | * Author: David de Regt 4 | * Copyright: Microsoft 2015 5 | * 6 | * Abstract helpers for all NoSqlProvider DbProviders that are based on SQL backings. 7 | */ 8 | 9 | import { ok } from 'assert'; 10 | import { 11 | repeat, map, isError, attempt, includes, filter, flatten, partition, 12 | intersection, some, keyBy, noop, isEqual, find, indexOf, fill, each 13 | } from 'lodash'; 14 | 15 | import { IDeferred, defer } from './defer'; 16 | import { getFullTextIndexWordsForItem, breakAndNormalizeSearchPhrase } from './FullTextSearchHelpers'; 17 | import { 18 | ItemType, IndexSchema, StoreSchema, DbProvider, 19 | DbIndex, DbSchema, DbStore, DbTransaction, QuerySortOrder, FullTextTermResolution 20 | } from './NoSqlProvider'; 21 | import { 22 | serializeKeyToString, isCompoundKeyPath, formListOfSerializedKeys, 23 | getSerializedKeyForKeypath, arrayify, getValueForSingleKeypath 24 | } from './NoSqlProviderUtils'; 25 | 26 | // Extending interfaces that should be in lib.d.ts but aren't for some reason. 27 | export interface SQLVoidCallback { 28 | (): void; 29 | } 30 | 31 | export interface SQLTransactionCallback { 32 | (transaction: SQLTransaction): void; 33 | } 34 | 35 | export interface SQLTransactionErrorCallback { 36 | (error: SQLError): void; 37 | } 38 | 39 | export interface SQLDatabase { 40 | version: string; 41 | 42 | changeVersion(oldVersion: string, newVersion: string, callback?: SQLTransactionCallback, 43 | errorCallback?: SQLTransactionErrorCallback, successCallback?: SQLVoidCallback): void; 44 | transaction(callback?: SQLTransactionCallback, errorCallback?: SQLTransactionErrorCallback, 45 | successCallback?: SQLVoidCallback): void; 46 | readTransaction(callback?: SQLTransactionCallback, errorCallback?: SQLTransactionErrorCallback, 47 | successCallback?: SQLVoidCallback): void; 48 | } 49 | 50 | const schemaVersionKey = 'schemaVersion'; 51 | 52 | // This was taked from the sqlite documentation 53 | const SQLITE_MAX_SQL_LENGTH_IN_BYTES = 1000000; 54 | 55 | const DB_SIZE_ESIMATE_DEFAULT = 200; 56 | const DB_MIGRATION_MAX_BYTE_TARGET = 1000000; 57 | 58 | interface IndexMetadata { 59 | key: string; 60 | storeName: string; 61 | index: IndexSchema; 62 | } 63 | 64 | function getIndexIdentifier(storeSchema: StoreSchema, index: IndexSchema): string { 65 | return storeSchema.name + '_' + index.name; 66 | } 67 | 68 | // Certain indexes use a separate table for pivot: 69 | // * Multientry indexes 70 | // * Full-text indexes that support FTS3 71 | function indexUsesSeparateTable(indexSchema: IndexSchema, supportsFTS3: boolean): boolean { 72 | return indexSchema.multiEntry || (!!indexSchema.fullText && supportsFTS3); 73 | } 74 | 75 | function generateParamPlaceholder(count: number): string { 76 | ok(count >= 1, 'Must provide at least one parameter to SQL statement'); 77 | // Generate correct count of ?'s and slice off trailing comma 78 | return repeat('?,', count).slice(0, -1); 79 | } 80 | 81 | const FakeFTSJoinToken = '^$^'; 82 | 83 | // Limit LIMIT numbers to a reasonable size to not break queries. 84 | const LimitMax = Math.pow(2, 32); 85 | 86 | export abstract class SqlProviderBase extends DbProvider { 87 | constructor(protected _supportsFTS3: boolean) { 88 | super(); 89 | // NOP 90 | } 91 | 92 | abstract openTransaction(storeNames: string[] | undefined, writeNeeded: boolean): Promise; 93 | 94 | private _getMetadata(trans: SqlTransaction): Promise<{ name: string; value: string; }[]> { 95 | // Create table if needed 96 | return trans.runQuery('CREATE TABLE IF NOT EXISTS metadata (name TEXT PRIMARY KEY, value TEXT)').then(() => { 97 | return trans.runQuery('SELECT name, value from metadata', []); 98 | }); 99 | } 100 | 101 | private _storeIndexMetadata(trans: SqlTransaction, meta: IndexMetadata) { 102 | return trans.runQuery('INSERT OR REPLACE into metadata (\'name\', \'value\') VALUES' + 103 | '(\'' + meta.key + '\', ?)', [JSON.stringify(meta)]); 104 | } 105 | 106 | private _getDbVersion(): Promise { 107 | return this.openTransaction(undefined, true).then(trans => { 108 | // Create table if needed 109 | return trans.runQuery('CREATE TABLE IF NOT EXISTS metadata (name TEXT PRIMARY KEY, value TEXT)').then(() => { 110 | return trans.runQuery('SELECT value from metadata where name=?', [schemaVersionKey]).then(data => { 111 | if (data && data[0] && data[0].value) { 112 | return Number(data[0].value) || 0; 113 | } 114 | return 0; 115 | }); 116 | }); 117 | }); 118 | } 119 | 120 | protected _changeDbVersion(oldVersion: number, newVersion: number): Promise { 121 | return this.openTransaction(undefined, true).then(trans => { 122 | return trans.runQuery('INSERT OR REPLACE into metadata (\'name\', \'value\') VALUES (\'' + schemaVersionKey + '\', ?)', 123 | [newVersion]) 124 | .then(() => trans); 125 | }); 126 | } 127 | 128 | protected _ourVersionChecker(wipeIfExists: boolean): Promise { 129 | return this._getDbVersion() 130 | .then(oldVersion => { 131 | if (oldVersion !== this._schema!!!.version) { 132 | // Needs a schema upgrade/change 133 | if (!wipeIfExists && this._schema!!!.version < oldVersion) { 134 | console.log('Database version too new (' + oldVersion + ') for schema version (' + this._schema!!!.version + 135 | '). Wiping!'); 136 | wipeIfExists = true; 137 | } 138 | 139 | return this._changeDbVersion(oldVersion, this._schema!!!.version).then(trans => { 140 | return this._upgradeDb(trans, oldVersion, wipeIfExists); 141 | }); 142 | } else if (wipeIfExists) { 143 | // No version change, but wipe anyway 144 | return this.openTransaction(undefined, true).then(trans => { 145 | return this._upgradeDb(trans, oldVersion, true); 146 | }); 147 | } 148 | return undefined; 149 | }); 150 | } 151 | 152 | protected _upgradeDb(trans: SqlTransaction, oldVersion: number, wipeAnyway: boolean): Promise { 153 | // Get a list of all tables, columns and indexes on the tables 154 | return this._getMetadata(trans).then(fullMeta => { 155 | // Get Index metadatas 156 | let indexMetadata: IndexMetadata[] = 157 | map(fullMeta, meta => { 158 | const metaObj = attempt(() => { 159 | return JSON.parse(meta.value); 160 | }); 161 | if (isError(metaObj)) { 162 | return undefined; 163 | } 164 | return metaObj; 165 | }) 166 | .filter(meta => !!meta && !!meta.storeName); 167 | 168 | return trans.runQuery('SELECT type, name, tbl_name, sql from sqlite_master', []) 169 | .then(rows => { 170 | let tableNames: string[] = []; 171 | let indexNames: { [table: string]: string[] } = {}; 172 | let indexTables: { [table: string]: string[] } = {}; 173 | let tableSqlStatements: { [table: string]: string } = {}; 174 | 175 | for (const row of rows) { 176 | const tableName = row['tbl_name']; 177 | // Ignore browser metadata tables for websql support 178 | if (tableName === '__WebKitDatabaseInfoTable__' || tableName === 'metadata') { 179 | continue; 180 | } 181 | // Ignore FTS-generated side tables 182 | const endsIn = (str: string, checkstr: string) => { 183 | const i = str.indexOf(checkstr); 184 | return i !== -1 && i === str.length - checkstr.length; 185 | }; 186 | if (endsIn(tableName, '_content') || endsIn(tableName, '_segments') || endsIn(tableName, '_segdir')) { 187 | continue; 188 | } 189 | if (row['type'] === 'table') { 190 | tableNames.push(row['name']); 191 | tableSqlStatements[row['name']] = row['sql']; 192 | const nameSplit = row['name'].split('_'); 193 | if (nameSplit.length === 1) { 194 | if (!indexNames[row['name']]) { 195 | indexNames[row['name']] = []; 196 | } 197 | if (!indexTables[row['name']]) { 198 | indexTables[row['name']] = []; 199 | } 200 | } else { 201 | const tableName = nameSplit[0]; 202 | if (indexTables[tableName]) { 203 | indexTables[tableName].push(nameSplit[1]); 204 | } else { 205 | indexTables[tableName] = [nameSplit[1]]; 206 | } 207 | } 208 | } 209 | if (row['type'] === 'index') { 210 | if (row['name'].substring(0, 17) === 'sqlite_autoindex_') { 211 | // auto-index, ignore 212 | continue; 213 | } 214 | if (!indexNames[tableName]) { 215 | indexNames[tableName] = []; 216 | } 217 | indexNames[tableName].push(row['name']); 218 | } 219 | } 220 | 221 | const deleteFromMeta = (metasToDelete: IndexMetadata[]) => { 222 | if (metasToDelete.length === 0) { 223 | return Promise.resolve([]); 224 | } 225 | 226 | // Generate as many '?' as there are params 227 | const placeholder = generateParamPlaceholder(metasToDelete.length); 228 | 229 | return trans.runQuery('DELETE FROM metadata WHERE name IN (' + placeholder + ')', 230 | map(metasToDelete, meta => meta.key)); 231 | }; 232 | 233 | // Check each table! 234 | let dropQueries: Promise[] = []; 235 | if (wipeAnyway || (this._schema!!!.lastUsableVersion && oldVersion < this._schema!!!.lastUsableVersion!!!)) { 236 | // Clear all stores if it's past the usable version 237 | if (!wipeAnyway) { 238 | console.log('Old version detected (' + oldVersion + '), clearing all tables'); 239 | } 240 | 241 | dropQueries = map(tableNames, name => trans.runQuery('DROP TABLE ' + name)); 242 | 243 | if (indexMetadata.length > 0) { 244 | // Drop all existing metadata 245 | dropQueries.push(deleteFromMeta(indexMetadata)); 246 | indexMetadata = []; 247 | } 248 | tableNames = []; 249 | } else { 250 | // Just delete tables we don't care about anymore. Preserve multi-entry tables, they may not be changed 251 | let tableNamesNeeded: string[] = []; 252 | for (const store of this._schema!!!.stores) { 253 | tableNamesNeeded.push(store.name); 254 | if (store.indexes) { 255 | for (const index of store.indexes) { 256 | if (indexUsesSeparateTable(index, this._supportsFTS3)) { 257 | tableNamesNeeded.push(getIndexIdentifier(store, index)); 258 | } 259 | } 260 | } 261 | } 262 | let tableNamesNotNeeded = filter(tableNames, name => !includes(tableNamesNeeded, name)); 263 | dropQueries = flatten(map(tableNamesNotNeeded, name => { 264 | const transList: Promise[] = [trans.runQuery('DROP TABLE ' + name)]; 265 | const metasToDelete = filter(indexMetadata, meta => meta.storeName === name); 266 | const metaKeysToDelete = map(metasToDelete, meta => meta.key); 267 | 268 | // Clean up metas 269 | if (metasToDelete.length > 0) { 270 | transList.push(deleteFromMeta(metasToDelete)); 271 | indexMetadata = filter(indexMetadata, meta => !includes(metaKeysToDelete, meta.key)); 272 | } 273 | return transList; 274 | })); 275 | 276 | tableNames = filter(tableNames, name => includes(tableNamesNeeded, name)); 277 | } 278 | 279 | const tableColumns: { [table: string]: string[] } = {}; 280 | 281 | const getColumnNames = (tableName: string): string[] => { 282 | // Try to get all the column names from SQL create statement 283 | const r = /CREATE\s+TABLE\s+\w+\s+\(([^\)]+)\)/; 284 | const columnPart = tableSqlStatements[tableName].match(r); 285 | if (columnPart) { 286 | return columnPart[1].split(',').map(p => p.trim().split(/\s+/)[0]); 287 | } 288 | return []; 289 | }; 290 | 291 | for (const table of tableNames) { 292 | tableColumns[table] = getColumnNames(table); 293 | } 294 | 295 | return Promise.all(dropQueries).then(() => { 296 | 297 | let tableQueries: Promise[] = []; 298 | 299 | // Go over each store and see what needs changing 300 | for (const storeSchema of this._schema!!!.stores) { 301 | 302 | // creates indexes for provided schemas 303 | const indexMaker = (indexes: IndexSchema[] = []) => { 304 | let metaQueries: Promise[] = []; 305 | const indexQueries = map(indexes, index => { 306 | const indexIdentifier = getIndexIdentifier(storeSchema, index); 307 | 308 | // Store meta for the index 309 | const newMeta: IndexMetadata = { 310 | key: indexIdentifier, 311 | storeName: storeSchema.name, 312 | index: index 313 | }; 314 | metaQueries.push(this._storeIndexMetadata(trans, newMeta)); 315 | // Go over each index and see if we need to create an index or a table for a multiEntry index 316 | if (index.multiEntry) { 317 | if (isCompoundKeyPath(index.keyPath)) { 318 | return Promise.reject('Can\'t use multiEntry and compound keys'); 319 | } else { 320 | return trans.runQuery('CREATE TABLE IF NOT EXISTS ' + indexIdentifier + 321 | ' (nsp_key TEXT, nsp_refpk TEXT' + 322 | (index.includeDataInIndex ? ', nsp_data TEXT' : '') + ')').then(() => { 323 | return trans.runQuery('CREATE ' + (index.unique ? 'UNIQUE ' : '') + 324 | 'INDEX IF NOT EXISTS ' + 325 | indexIdentifier + '_pi ON ' + indexIdentifier + ' (nsp_key, nsp_refpk' + 326 | (index.includeDataInIndex ? ', nsp_data' : '') + ')'); 327 | }); 328 | } 329 | } else if (index.fullText && this._supportsFTS3) { 330 | // If FTS3 isn't supported, we'll make a normal column and use LIKE to seek over it, so the 331 | // fallback below works fine. 332 | return trans.runQuery('CREATE VIRTUAL TABLE IF NOT EXISTS ' + indexIdentifier + 333 | ' USING FTS3(nsp_key TEXT, nsp_refpk TEXT)'); 334 | } else { 335 | return trans.runQuery('CREATE ' + (index.unique ? 'UNIQUE ' : '') + 336 | 'INDEX IF NOT EXISTS ' + indexIdentifier + 337 | ' ON ' + storeSchema.name + ' (nsp_i_' + index.name + 338 | (index.includeDataInIndex ? ', nsp_data' : '') + ')'); 339 | } 340 | }); 341 | 342 | return Promise.all(indexQueries.concat(metaQueries)); 343 | }; 344 | 345 | // Form SQL statement for table creation 346 | let fieldList = []; 347 | 348 | fieldList.push('nsp_pk TEXT PRIMARY KEY'); 349 | fieldList.push('nsp_data TEXT'); 350 | 351 | const columnBasedIndices = filter(storeSchema.indexes, index => 352 | !indexUsesSeparateTable(index, this._supportsFTS3)); 353 | 354 | const indexColumnsNames = map(columnBasedIndices, index => 'nsp_i_' + index.name + ' TEXT'); 355 | fieldList = fieldList.concat(indexColumnsNames); 356 | const tableMakerSql = 'CREATE TABLE ' + storeSchema.name + ' (' + fieldList.join(', ') + ')'; 357 | 358 | const currentIndexMetas = filter(indexMetadata, meta => meta.storeName === storeSchema.name); 359 | 360 | const indexIdentifierDictionary = keyBy(storeSchema.indexes, index => getIndexIdentifier(storeSchema, index)); 361 | const indexMetaDictionary = keyBy(currentIndexMetas, meta => meta.key); 362 | 363 | // find which indices in the schema existed / did not exist before 364 | const [newIndices, existingIndices] = partition(storeSchema.indexes, index => 365 | !indexMetaDictionary[getIndexIdentifier(storeSchema, index)]); 366 | 367 | const existingIndexColumns = intersection(existingIndices, columnBasedIndices); 368 | 369 | // find indices in the meta that do not exist in the new schema 370 | const allRemovedIndexMetas = filter(currentIndexMetas, meta => 371 | !indexIdentifierDictionary[meta.key]); 372 | 373 | const [removedTableIndexMetas, removedColumnIndexMetas] = partition(allRemovedIndexMetas, 374 | meta => indexUsesSeparateTable(meta.index, this._supportsFTS3)); 375 | 376 | // find new indices which don't require backfill 377 | const newNoBackfillIndices = filter(newIndices, index => { 378 | return !!index.doNotBackfill; 379 | }); 380 | 381 | // columns requiring no backfill could be simply added to the table 382 | const newIndexColumnsNoBackfill = intersection(newNoBackfillIndices, columnBasedIndices); 383 | 384 | const columnAdder = () => { 385 | const addQueries = map(newIndexColumnsNoBackfill, index => 386 | trans.runQuery('ALTER TABLE ' + storeSchema.name + ' ADD COLUMN ' + 'nsp_i_' + index.name + ' TEXT') 387 | ); 388 | 389 | return Promise.all(addQueries); 390 | }; 391 | 392 | const tableMaker = () => { 393 | // Create the table 394 | return trans.runQuery(tableMakerSql) 395 | .then(() => indexMaker(storeSchema.indexes)); 396 | }; 397 | 398 | const columnExists = (tableName: string, columnName: string) => { 399 | return includes(tableColumns[tableName], columnName); 400 | }; 401 | 402 | const needsFullMigration = () => { 403 | // Check all the indices in the schema 404 | return some(storeSchema.indexes, index => { 405 | const indexIdentifier = getIndexIdentifier(storeSchema, index); 406 | const indexMeta = indexMetaDictionary[indexIdentifier]; 407 | 408 | // if there's a new index that doesn't require backfill, continue 409 | // If there's a new index that requires backfill - we need to migrate 410 | if (!indexMeta) { 411 | return !index.doNotBackfill; 412 | } 413 | 414 | // If the index schemas don't match - we need to migrate 415 | if (!isEqual(indexMeta.index, index)) { 416 | return true; 417 | } 418 | 419 | // Check that indicies actually exist in the right place 420 | if (indexUsesSeparateTable(index, this._supportsFTS3)) { 421 | if (!includes(tableNames, indexIdentifier)) { 422 | return true; 423 | } 424 | } else { 425 | if (!columnExists(storeSchema.name, 'nsp_i_' + index.name)) { 426 | return true; 427 | } 428 | } 429 | 430 | return false; 431 | }); 432 | }; 433 | 434 | const dropColumnIndices = () => { 435 | return map(indexNames[storeSchema.name], indexName => 436 | trans.runQuery('DROP INDEX ' + indexName)); 437 | }; 438 | 439 | const dropIndexTables = (tableNames: string[]) => { 440 | return map(tableNames, tableName => 441 | trans.runQuery('DROP TABLE IF EXISTS ' + storeSchema.name + '_' + tableName) 442 | ); 443 | }; 444 | 445 | const createTempTable = () => { 446 | // Then rename the table to a temp_[name] table so we can migrate the data out of it 447 | return trans.runQuery('ALTER TABLE ' + storeSchema.name + ' RENAME TO temp_' + storeSchema.name); 448 | }; 449 | 450 | const dropTempTable = () => { 451 | return trans.runQuery('DROP TABLE temp_' + storeSchema.name); 452 | }; 453 | 454 | // find is there are some columns that should be, but are not indices 455 | // this is to fix a mismatch between the schema in metadata and the actual table state 456 | const someIndicesMissing = some(columnBasedIndices, index => 457 | columnExists(storeSchema.name, 'nsp_i_' + index.name) 458 | && !includes(indexNames[storeSchema.name], getIndexIdentifier(storeSchema, index)) 459 | ); 460 | 461 | // If the table exists, check if we can to determine if a migration is needed 462 | // If a full migration is needed, we have to copy all the data over and re-populate indices 463 | // If a in-place migration is enough, we can just copy the data 464 | // If no migration is needed, we can just add new column for new indices 465 | const tableExists = includes(tableNames, storeSchema.name); 466 | const doFullMigration = tableExists && needsFullMigration(); 467 | const doSqlInPlaceMigration = tableExists && !doFullMigration && removedColumnIndexMetas.length > 0; 468 | const adddNewColumns = tableExists && !doFullMigration && !doSqlInPlaceMigration 469 | && newNoBackfillIndices.length > 0; 470 | const recreateIndices = tableExists && !doFullMigration && !doSqlInPlaceMigration && someIndicesMissing; 471 | 472 | const indexFixer = () => { 473 | if (recreateIndices) { 474 | return indexMaker(storeSchema.indexes); 475 | } 476 | return Promise.resolve([]); 477 | }; 478 | 479 | const indexTableAndMetaDropper = () => { 480 | const indexTablesToDrop = doFullMigration 481 | ? indexTables[storeSchema.name] : removedTableIndexMetas.map(meta => meta.key); 482 | return Promise.all([deleteFromMeta(allRemovedIndexMetas), ...dropIndexTables(indexTablesToDrop)]); 483 | }; 484 | 485 | if (!tableExists) { 486 | // Table doesn't exist -- just go ahead and create it without the migration path 487 | tableQueries.push(tableMaker()); 488 | } 489 | 490 | if (doFullMigration) { 491 | // Migrate the data over using our existing put functions 492 | // (since it will do the right things with the indexes) 493 | // and delete the temp table. 494 | const jsMigrator = (batchOffset = 0): Promise => { 495 | let esimatedSize = storeSchema.estimatedObjBytes || DB_SIZE_ESIMATE_DEFAULT; 496 | let batchSize = Math.max(1, Math.floor(DB_MIGRATION_MAX_BYTE_TARGET / esimatedSize)); 497 | let store = trans.getStore(storeSchema.name); 498 | return trans.internal_getResultsFromQuery('SELECT nsp_data FROM temp_' + storeSchema.name + ' LIMIT ' + 499 | batchSize + ' OFFSET ' + batchOffset) 500 | .then(objs => { 501 | return store.put(objs).then(() => { 502 | // Are we done migrating? 503 | if (objs.length < batchSize) { 504 | return undefined; 505 | } 506 | return jsMigrator(batchOffset + batchSize); 507 | }); 508 | }); 509 | }; 510 | 511 | tableQueries.push( 512 | Promise.all([ 513 | indexTableAndMetaDropper(), 514 | dropColumnIndices(), 515 | ]) 516 | .then(createTempTable) 517 | .then(tableMaker) 518 | .then(() => { 519 | return jsMigrator(); 520 | }) 521 | .then(dropTempTable) 522 | ); 523 | } 524 | 525 | if (doSqlInPlaceMigration) { 526 | const sqlInPlaceMigrator = () => { 527 | const columnsToCopy = ['nsp_pk', 'nsp_data', 528 | ...map(existingIndexColumns, index => 'nsp_i_' + index.name) 529 | ].join(', '); 530 | 531 | return trans.runQuery('INSERT INTO ' + storeSchema.name + ' (' + columnsToCopy + ')' + 532 | ' SELECT ' + columnsToCopy + 533 | ' FROM temp_' + storeSchema.name); 534 | }; 535 | 536 | tableQueries.push( 537 | Promise.all([ 538 | indexTableAndMetaDropper(), 539 | dropColumnIndices(), 540 | ]) 541 | .then(createTempTable) 542 | .then(tableMaker) 543 | .then(sqlInPlaceMigrator) 544 | .then(dropTempTable) 545 | ); 546 | 547 | } 548 | 549 | if (adddNewColumns) { 550 | const newIndexMaker = () => indexMaker(newNoBackfillIndices); 551 | 552 | tableQueries.push( 553 | indexTableAndMetaDropper(), 554 | columnAdder() 555 | .then(newIndexMaker) 556 | .then(indexFixer) 557 | ); 558 | } else if (recreateIndices) { 559 | tableQueries.push(indexFixer()); 560 | } 561 | 562 | } 563 | 564 | return Promise.all(tableQueries); 565 | }); 566 | }); 567 | }).then(noop); 568 | } 569 | } 570 | 571 | // The DbTransaction implementation for the WebSQL DbProvider. All WebSQL accesses go through the transaction 572 | // object, so this class actually has several helpers for executing SQL queries, getting results from them, etc. 573 | export abstract class SqlTransaction implements DbTransaction { 574 | private _isOpen = true; 575 | 576 | constructor( 577 | protected _schema: DbSchema, 578 | protected _verbose: boolean, 579 | protected _maxVariables: number, 580 | private _supportsFTS3: boolean) { 581 | if (this._verbose) { 582 | console.log('Opening Transaction'); 583 | } 584 | } 585 | 586 | protected _isTransactionOpen(): boolean { 587 | return this._isOpen; 588 | } 589 | 590 | internal_markTransactionClosed(): void { 591 | if (this._verbose) { 592 | console.log('Marking Transaction Closed'); 593 | } 594 | this._isOpen = false; 595 | } 596 | 597 | abstract getCompletionPromise(): Promise; 598 | abstract abort(): void; 599 | 600 | abstract runQuery(sql: string, parameters?: any[]): Promise; 601 | 602 | internal_getMaxVariables(): number { 603 | return this._maxVariables; 604 | } 605 | 606 | internal_nonQuery(sql: string, parameters?: any[]): Promise { 607 | return this.runQuery(sql, parameters).then(noop); 608 | } 609 | 610 | internal_getResultsFromQuery(sql: string, parameters?: any[]): Promise { 611 | return this.runQuery(sql, parameters).then(rows => { 612 | let rets: T[] = []; 613 | for (let i = 0; i < rows.length; i++) { 614 | try { 615 | rets.push(JSON.parse(rows[i].nsp_data)); 616 | } catch (e) { 617 | return Promise.reject('Error parsing database entry in getResultsFromQuery: ' + JSON.stringify(rows[i].nsp_data)); 618 | } 619 | } 620 | return rets; 621 | }); 622 | } 623 | 624 | internal_getResultFromQuery(sql: string, parameters?: any[]): Promise { 625 | return this.internal_getResultsFromQuery(sql, parameters) 626 | .then(rets => rets.length < 1 ? undefined : rets[0]); 627 | } 628 | 629 | getStore(storeName: string): DbStore { 630 | const storeSchema = find(this._schema.stores, store => store.name === storeName); 631 | if (!storeSchema) { 632 | throw new Error('Store not found: ' + storeName); 633 | } 634 | 635 | return new SqlStore(this, storeSchema, this._requiresUnicodeReplacement(), this._supportsFTS3, this._verbose); 636 | } 637 | 638 | markCompleted(): void { 639 | // noop 640 | } 641 | 642 | protected _requiresUnicodeReplacement(): boolean { 643 | return false; 644 | } 645 | } 646 | 647 | export interface SQLError { 648 | code: number; 649 | message: string; 650 | } 651 | 652 | export interface SQLResultSet { 653 | insertId: number; 654 | rowsAffected: number; 655 | rows: SQLResultSetRowList; 656 | } 657 | 658 | export interface SQLResultSetRowList { 659 | length: number; 660 | item(index: number): any; 661 | } 662 | 663 | export interface SQLStatementCallback { 664 | (transaction: SQLTransaction, resultSet: SQLResultSet): void; 665 | } 666 | 667 | export interface SQLStatementErrorCallback { 668 | (transaction: SQLTransaction, error: SQLError): void; 669 | } 670 | 671 | export interface SQLTransaction { 672 | executeSql(sqlStatement: string, args?: any[], callback?: SQLStatementCallback, errorCallback?: SQLStatementErrorCallback): void; 673 | } 674 | 675 | // Generic base transaction for anything that matches the syntax of a SQLTransaction interface for executing sql commands. 676 | // Conveniently, this works for both WebSql and cordova's Sqlite plugin. 677 | export abstract class SqliteSqlTransaction extends SqlTransaction { 678 | private _pendingQueries: IDeferred[] = []; 679 | 680 | constructor(protected _trans: SQLTransaction, schema: DbSchema, verbose: boolean, maxVariables: number, 681 | supportsFTS3: boolean) { 682 | super(schema, verbose, maxVariables, supportsFTS3); 683 | } 684 | 685 | abstract getErrorHandlerReturnValue(): boolean; 686 | 687 | // If an external provider of the transaction determines that the transaction has failed but won't report its failures 688 | // (i.e. in the case of WebSQL), we need a way to kick the hanging queries that they're going to fail since otherwise 689 | // they'll never respond. 690 | failAllPendingQueries(error: any) { 691 | const list = this._pendingQueries; 692 | this._pendingQueries = []; 693 | for (const query of list) { 694 | query.reject(error); 695 | } 696 | } 697 | 698 | runQuery(sql: string, parameters?: any[]): Promise { 699 | if (!this._isTransactionOpen()) { 700 | return Promise.reject('SqliteSqlTransaction already closed'); 701 | } 702 | 703 | const deferred = defer(); 704 | this._pendingQueries.push(deferred); 705 | 706 | let startTime: number; 707 | if (this._verbose) { 708 | startTime = Date.now(); 709 | } 710 | 711 | const errRet = attempt(() => { 712 | this._trans.executeSql(sql, parameters, (t, rs) => { 713 | const index = indexOf(this._pendingQueries, deferred); 714 | if (index !== -1) { 715 | let rows = []; 716 | for (let i = 0; i < rs.rows.length; i++) { 717 | rows.push(rs.rows.item(i)); 718 | } 719 | this._pendingQueries.splice(index, 1); 720 | deferred.resolve(rows); 721 | } else { 722 | console.error('SQL statement resolved twice (success this time): ' + sql); 723 | } 724 | }, (t, err) => { 725 | if (!err) { 726 | // The cordova-native-sqlite-storage plugin only passes a single parameter here, the error, 727 | // slightly breaking the interface. 728 | err = t as any; 729 | } 730 | 731 | const index = indexOf(this._pendingQueries, deferred); 732 | if (index !== -1) { 733 | this._pendingQueries.splice(index, 1); 734 | deferred.reject(err); 735 | } else { 736 | console.error('SQL statement resolved twice (this time with failure)'); 737 | } 738 | 739 | return this.getErrorHandlerReturnValue(); 740 | }); 741 | }); 742 | 743 | if (errRet) { 744 | deferred.reject(errRet); 745 | } 746 | 747 | let promise = deferred.promise; 748 | if (this._verbose) { 749 | promise = promise.finally(() => { 750 | console.log('SqlTransaction RunQuery: (' + (Date.now() - startTime) + 'ms): SQL: ' + sql); 751 | }); 752 | } 753 | return promise; 754 | } 755 | } 756 | 757 | // DbStore implementation for the SQL-based DbProviders. Implements the getters/setters against the transaction object and all of the 758 | // glue for index/compound key support. 759 | class SqlStore implements DbStore { 760 | constructor(private _trans: SqlTransaction, private _schema: StoreSchema, private _replaceUnicode: boolean, 761 | private _supportsFTS3: boolean, private _verbose: boolean) { 762 | // Empty 763 | } 764 | 765 | get(key: KeyType): Promise { 766 | const joinedKey = attempt(() => { 767 | return serializeKeyToString(key, this._schema.primaryKeyPath); 768 | }); 769 | if (isError(joinedKey)) { 770 | return Promise.reject(joinedKey); 771 | } 772 | 773 | let startTime: number; 774 | if (this._verbose) { 775 | startTime = Date.now(); 776 | } 777 | 778 | let promise = this._trans.internal_getResultFromQuery('SELECT nsp_data FROM ' + this._schema.name + 779 | ' WHERE nsp_pk = ?', [joinedKey]); 780 | if (this._verbose) { 781 | promise = promise.finally(() => { 782 | console.log('SqlStore (' + this._schema.name + ') get: (' + (Date.now() - startTime) + 'ms)'); 783 | }); 784 | } 785 | return promise; 786 | } 787 | 788 | getMultiple(keyOrKeys: KeyType | KeyType[]): Promise { 789 | const joinedKeys = attempt(() => { 790 | return formListOfSerializedKeys(keyOrKeys, this._schema.primaryKeyPath); 791 | }); 792 | if (isError(joinedKeys)) { 793 | return Promise.reject(joinedKeys); 794 | } 795 | 796 | if (joinedKeys.length === 0) { 797 | return Promise.resolve([]); 798 | } 799 | 800 | let startTime: number; 801 | if (this._verbose) { 802 | startTime = Date.now(); 803 | } 804 | 805 | let promise = this._trans.internal_getResultsFromQuery('SELECT nsp_data FROM ' + this._schema.name + ' WHERE nsp_pk IN (' + 806 | generateParamPlaceholder(joinedKeys.length) + ')', joinedKeys); 807 | if (this._verbose) { 808 | promise = promise.finally(() => { 809 | console.log('SqlStore (' + this._schema.name + ') getMultiple: (' + (Date.now() - startTime) + 'ms): Count: ' + 810 | joinedKeys.length); 811 | }); 812 | } 813 | return promise; 814 | } 815 | 816 | private static _unicodeFixer = new RegExp('[\u2028\u2029]', 'g'); 817 | 818 | put(itemOrItems: ItemType | ItemType[]): Promise { 819 | let items = arrayify(itemOrItems); 820 | 821 | if (items.length === 0) { 822 | return Promise.resolve(undefined); 823 | } 824 | 825 | let startTime: number; 826 | if (this._verbose) { 827 | startTime = Date.now(); 828 | } 829 | 830 | let fields: string[] = ['nsp_pk', 'nsp_data']; 831 | let qmarks: string[] = ['?', '?']; 832 | let args: any[] = []; 833 | 834 | if (this._schema.indexes) { 835 | for (const index of this._schema.indexes) { 836 | if (!indexUsesSeparateTable(index, this._supportsFTS3)) { 837 | qmarks.push('?'); 838 | fields.push('nsp_i_' + index.name); 839 | } 840 | } 841 | } 842 | 843 | const qmarkString = qmarks.join(','); 844 | const datas = attempt(() => { 845 | return map(items, (item) => { 846 | let serializedData = JSON.stringify(item); 847 | // For now, until an issue with cordova-ios is fixed (https://issues.apache.org/jira/browse/CB-9435), have to replace 848 | // \u2028 and 2029 with blanks because otherwise the command boundary with cordova-ios silently eats any strings with them. 849 | if (this._replaceUnicode) { 850 | serializedData = serializedData.replace(SqlStore._unicodeFixer, ''); 851 | } 852 | args.push(getSerializedKeyForKeypath(item, this._schema.primaryKeyPath), serializedData); 853 | 854 | if (this._schema.indexes) { 855 | for (const index of this._schema.indexes) { 856 | if (indexUsesSeparateTable(index, this._supportsFTS3)) { 857 | continue; 858 | } 859 | 860 | if (index.fullText && !this._supportsFTS3) { 861 | args.push(FakeFTSJoinToken + 862 | getFullTextIndexWordsForItem(index.keyPath, item).join(FakeFTSJoinToken)); 863 | } else if (!index.multiEntry) { 864 | args.push(getSerializedKeyForKeypath(item, index.keyPath)); 865 | } 866 | } 867 | } 868 | 869 | return serializedData; 870 | }); 871 | }); 872 | if (isError(datas)) { 873 | return Promise.reject(datas); 874 | } 875 | 876 | // Need to not use too many variables per insert, so batch the insert if needed. 877 | let queries: Promise[] = []; 878 | const itemPageSize = Math.floor(this._trans.internal_getMaxVariables() / fields.length); 879 | for (let i = 0; i < items.length; i += itemPageSize) { 880 | const thisPageCount = Math.min(itemPageSize, items.length - i); 881 | const qmarksValues = fill(new Array(thisPageCount), qmarkString); 882 | queries.push(this._trans.internal_nonQuery('INSERT OR REPLACE INTO ' + this._schema.name + ' (' + fields.join(',') + 883 | ') VALUES (' + qmarksValues.join('),(') + ')', args.splice(0, thisPageCount * fields.length))); 884 | } 885 | 886 | // Also prepare mulltiEntry and FullText indexes 887 | if (some(this._schema.indexes, index => indexUsesSeparateTable(index, this._supportsFTS3))) { 888 | const keysToDeleteByIndex: { [indexIndex: number]: string[] } = {}; 889 | const dataToInsertByIndex: { [indexIndex: number]: string[] } = {}; 890 | 891 | // Track 0-based index of item we're looking at 892 | let itemIndex = -1; 893 | for (const item of items) { 894 | itemIndex++; 895 | const key = attempt(() => { 896 | return getSerializedKeyForKeypath(item, this._schema.primaryKeyPath)!!!; 897 | }); 898 | if (isError(key)) { 899 | queries.push(Promise.reject(key)); 900 | continue; 901 | } 902 | 903 | if (this._schema.indexes) { 904 | // Track 0-based index of index we're looking at 905 | let indexIndex = -1; 906 | for (const index of this._schema.indexes) { 907 | indexIndex++; 908 | let serializedKeys: string[]; 909 | 910 | if (index.fullText && this._supportsFTS3) { 911 | // FTS3 terms go in a separate virtual table... 912 | serializedKeys = [getFullTextIndexWordsForItem(index.keyPath, item).join(' ')]; 913 | } else if (index.multiEntry) { 914 | // Have to extract the multiple entries into the alternate table... 915 | const valsRaw = getValueForSingleKeypath(item, index.keyPath); 916 | if (valsRaw) { 917 | const serializedKeysOrErr = attempt(() => { 918 | return map(arrayify(valsRaw), val => 919 | serializeKeyToString(val, index.keyPath)); 920 | }); 921 | if (isError(serializedKeysOrErr)) { 922 | queries.push(Promise.reject(serializedKeysOrErr)); 923 | continue; 924 | } 925 | serializedKeys = serializedKeysOrErr; 926 | } else { 927 | serializedKeys = []; 928 | } 929 | } else { 930 | continue; 931 | } 932 | 933 | // Capture insert data 934 | if (serializedKeys.length > 0) { 935 | if (!dataToInsertByIndex[indexIndex]) { 936 | dataToInsertByIndex[indexIndex] = []; 937 | } 938 | const dataToInsert = dataToInsertByIndex[indexIndex]; 939 | for (const val of serializedKeys) { 940 | dataToInsert.push(val); 941 | dataToInsert.push(key); 942 | if (index.includeDataInIndex) { 943 | dataToInsert.push(datas[itemIndex]); 944 | } 945 | } 946 | } 947 | 948 | // Capture delete keys 949 | if (!keysToDeleteByIndex[indexIndex]) { 950 | keysToDeleteByIndex[indexIndex] = []; 951 | } 952 | 953 | keysToDeleteByIndex[indexIndex].push(key); 954 | } 955 | } 956 | } 957 | 958 | const deleteQueries: Promise[] = []; 959 | 960 | each(keysToDeleteByIndex, (keysToDelete, indedIndex) => { 961 | // We know indexes are defined if we have data to insert for them 962 | // each spits dictionary keys out as string, needs to turn into a number 963 | const index = this._schema.indexes!!![Number(indedIndex)]; 964 | const itemPageSize = this._trans.internal_getMaxVariables(); 965 | for (let i = 0; i < keysToDelete.length; i += itemPageSize) { 966 | const thisPageCount = Math.min(itemPageSize, keysToDelete.length - i); 967 | deleteQueries.push(this._trans.internal_nonQuery('DELETE FROM ' + this._schema.name + '_' + index.name + 968 | ' WHERE nsp_refpk IN (' + generateParamPlaceholder(thisPageCount) + ')', keysToDelete.splice(0, thisPageCount))); 969 | } 970 | }); 971 | 972 | // Delete and insert tracking - cannot insert until delete is completed 973 | queries.push(Promise.all(deleteQueries).then(() => { 974 | const insertQueries: Promise[] = []; 975 | each(dataToInsertByIndex, (data, indexIndex) => { 976 | // We know indexes are defined if we have data to insert for them 977 | // each spits dictionary keys out as string, needs to turn into a number 978 | const index = this._schema.indexes!!![Number(indexIndex)]; 979 | const insertParamCount = index.includeDataInIndex ? 3 : 2; 980 | const itemPageSize = Math.floor(this._trans.internal_getMaxVariables() / insertParamCount); 981 | // data contains all the input parameters 982 | for (let i = 0; i < (data.length / insertParamCount); i += itemPageSize) { 983 | const thisPageCount = Math.min(itemPageSize, (data.length / insertParamCount) - i); 984 | const qmarksValues = fill(new Array(thisPageCount), generateParamPlaceholder(insertParamCount)); 985 | insertQueries.push(this._trans.internal_nonQuery('INSERT INTO ' + 986 | this._schema.name + '_' + index.name + ' (nsp_key, nsp_refpk' + (index.includeDataInIndex ? ', nsp_data' : '') + 987 | ') VALUES ' + '(' + qmarksValues.join('),(') + ')', data.splice(0, thisPageCount * insertParamCount))); 988 | } 989 | }); 990 | return Promise.all(insertQueries).then(noop); 991 | })); 992 | 993 | } 994 | 995 | let promise = Promise.all(queries); 996 | if (this._verbose) { 997 | promise = promise.finally(() => { 998 | console.log('SqlStore (' + this._schema.name + ') put: (' + (Date.now() - startTime) + 'ms): Count: ' + items.length); 999 | }); 1000 | } 1001 | return promise.then(noop); 1002 | } 1003 | 1004 | remove(keyOrKeys: KeyType | KeyType[]): Promise { 1005 | const joinedKeys = attempt(() => { 1006 | return formListOfSerializedKeys(keyOrKeys, this._schema.primaryKeyPath); 1007 | }); 1008 | if (isError(joinedKeys)) { 1009 | return Promise.reject(joinedKeys); 1010 | } 1011 | 1012 | let startTime: number; 1013 | if (this._verbose) { 1014 | startTime = Date.now(); 1015 | } 1016 | 1017 | // Partition the parameters 1018 | var arrayOfParams: Array> = [[]]; 1019 | var totalLength = 0; 1020 | var totalItems = 0; 1021 | var partitionIndex = 0; 1022 | joinedKeys.forEach(joinedKey => { 1023 | 1024 | // Append the new item to the current partition 1025 | arrayOfParams[partitionIndex].push(joinedKey); 1026 | 1027 | // Accumulate the length 1028 | totalLength += joinedKey.length + 2; 1029 | 1030 | totalItems++; 1031 | 1032 | // Make sure we don't exceed the following sqlite limits, if so go to the next partition 1033 | let didReachSqlStatementLimit = totalLength > (SQLITE_MAX_SQL_LENGTH_IN_BYTES - 200); 1034 | let didExceedMaxVariableCount = totalItems >= this._trans.internal_getMaxVariables(); 1035 | if (didReachSqlStatementLimit || didExceedMaxVariableCount) { 1036 | totalLength = 0; 1037 | totalItems = 0; 1038 | partitionIndex++; 1039 | arrayOfParams.push(new Array()); 1040 | } 1041 | }); 1042 | 1043 | const queries = map(arrayOfParams, params => { 1044 | let queries: Promise[] = []; 1045 | 1046 | if (params.length === 0) { 1047 | return undefined; 1048 | } 1049 | 1050 | // Generate as many '?' as there are params 1051 | const placeholder = generateParamPlaceholder(params.length); 1052 | 1053 | if (this._schema.indexes) { 1054 | for (const index of this._schema.indexes) { 1055 | if (indexUsesSeparateTable(index, this._supportsFTS3)) { 1056 | queries.push(this._trans.internal_nonQuery('DELETE FROM ' + this._schema.name + '_' + index.name + 1057 | ' WHERE nsp_refpk IN (' + placeholder + ')', params)); 1058 | } 1059 | } 1060 | } 1061 | 1062 | queries.push(this._trans.internal_nonQuery('DELETE FROM ' + this._schema.name + 1063 | ' WHERE nsp_pk IN (' + placeholder + ')', params)); 1064 | 1065 | return Promise.all(queries).then(noop); 1066 | }); 1067 | 1068 | let promise = Promise.all(queries).then(noop); 1069 | if (this._verbose) { 1070 | promise = promise.finally(() => { 1071 | console.log('SqlStore (' + this._schema.name + ') remove: (' + (Date.now() - startTime) + 'ms): Count: ' + 1072 | joinedKeys.length); 1073 | }); 1074 | } 1075 | return promise; 1076 | } 1077 | 1078 | openIndex(indexName: string): DbIndex { 1079 | const indexSchema = find(this._schema.indexes, index => index.name === indexName); 1080 | if (!indexSchema) { 1081 | throw new Error('Index not found: ' + indexName); 1082 | } 1083 | 1084 | return new SqlStoreIndex(this._trans, this._schema, indexSchema, this._supportsFTS3, this._verbose); 1085 | } 1086 | 1087 | openPrimaryKey(): DbIndex { 1088 | return new SqlStoreIndex(this._trans, this._schema, undefined, this._supportsFTS3, this._verbose); 1089 | } 1090 | 1091 | clearAllData(): Promise { 1092 | let indexes = filter(this._schema.indexes, index => indexUsesSeparateTable(index, this._supportsFTS3)); 1093 | let queries = map(indexes, index => 1094 | this._trans.internal_nonQuery('DELETE FROM ' + this._schema.name + '_' + index.name)); 1095 | 1096 | queries.push(this._trans.internal_nonQuery('DELETE FROM ' + this._schema.name)); 1097 | 1098 | return Promise.all(queries).then(noop); 1099 | } 1100 | } 1101 | 1102 | // DbIndex implementation for SQL-based DbProviders. Wraps all of the nasty compound key logic and general index traversal logic into 1103 | // the appropriate SQL queries. 1104 | class SqlStoreIndex implements DbIndex { 1105 | private _queryColumn: string; 1106 | private _tableName: string; 1107 | private _rawTableName: string; 1108 | private _indexTableName: string; 1109 | private _keyPath: string | string[]; 1110 | 1111 | constructor(protected _trans: SqlTransaction, storeSchema: StoreSchema, indexSchema: IndexSchema | undefined, 1112 | private _supportsFTS3: boolean, private _verbose: boolean) { 1113 | if (!indexSchema) { 1114 | // Going against the PK of the store 1115 | this._tableName = storeSchema.name; 1116 | this._rawTableName = this._tableName; 1117 | this._indexTableName = this._tableName; 1118 | this._queryColumn = 'nsp_pk'; 1119 | this._keyPath = storeSchema.primaryKeyPath; 1120 | } else { 1121 | if (indexUsesSeparateTable(indexSchema, this._supportsFTS3)) { 1122 | if (indexSchema.includeDataInIndex) { 1123 | this._tableName = storeSchema.name + '_' + indexSchema.name; 1124 | this._rawTableName = storeSchema.name; 1125 | this._indexTableName = storeSchema.name + '_' + indexSchema.name; 1126 | this._queryColumn = 'nsp_key'; 1127 | } else { 1128 | this._tableName = storeSchema.name + '_' + indexSchema.name + ' mi LEFT JOIN ' + storeSchema.name + 1129 | ' ON mi.nsp_refpk = ' + storeSchema.name + '.nsp_pk'; 1130 | this._rawTableName = storeSchema.name; 1131 | this._indexTableName = storeSchema.name + '_' + indexSchema.name; 1132 | this._queryColumn = 'mi.nsp_key'; 1133 | } 1134 | } else { 1135 | this._tableName = storeSchema.name; 1136 | this._rawTableName = this._tableName; 1137 | this._indexTableName = this._tableName; 1138 | this._queryColumn = 'nsp_i_' + indexSchema.name; 1139 | } 1140 | this._keyPath = indexSchema.keyPath; 1141 | } 1142 | } 1143 | 1144 | private _handleQuery(sql: string, args?: any[], reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, 1145 | offset?: number): Promise { 1146 | // Check if we must do some sort of ordering 1147 | if (reverseOrSortOrder !== QuerySortOrder.None) { 1148 | const reverse = reverseOrSortOrder === true || reverseOrSortOrder === QuerySortOrder.Reverse; 1149 | sql += ' ORDER BY ' + this._queryColumn + (reverse ? ' DESC' : ' ASC'); 1150 | } 1151 | 1152 | if (limit) { 1153 | if (limit > LimitMax) { 1154 | if (this._verbose) { 1155 | console.warn('Limit exceeded in _handleQuery (' + limit + ')'); 1156 | } 1157 | 1158 | limit = LimitMax; 1159 | } 1160 | sql += ' LIMIT ' + limit.toString(); 1161 | } 1162 | if (offset) { 1163 | sql += ' OFFSET ' + offset.toString(); 1164 | } 1165 | 1166 | return this._trans.internal_getResultsFromQuery(sql, args); 1167 | } 1168 | 1169 | getAll(reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number): Promise { 1170 | let startTime: number; 1171 | if (this._verbose) { 1172 | startTime = Date.now(); 1173 | } 1174 | 1175 | let promise = this._handleQuery('SELECT nsp_data FROM ' + this._tableName, undefined, reverseOrSortOrder, limit, offset); 1176 | if (this._verbose) { 1177 | promise = promise.finally(() => { 1178 | console.log('SqlStoreIndex (' + this._rawTableName + '/' + this._indexTableName + ') getAll: (' + 1179 | (Date.now() - startTime) + 'ms)'); 1180 | }); 1181 | } 1182 | return promise; 1183 | } 1184 | 1185 | getOnly(key: KeyType, reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number) 1186 | : Promise { 1187 | const joinedKey = attempt(() => { 1188 | return serializeKeyToString(key, this._keyPath); 1189 | }); 1190 | if (isError(joinedKey)) { 1191 | return Promise.reject(joinedKey); 1192 | } 1193 | 1194 | let startTime: number; 1195 | if (this._verbose) { 1196 | startTime = Date.now(); 1197 | } 1198 | 1199 | let promise = this._handleQuery('SELECT nsp_data FROM ' + this._tableName + ' WHERE ' + this._queryColumn + ' = ?', 1200 | [joinedKey], reverseOrSortOrder, limit, offset); 1201 | if (this._verbose) { 1202 | promise = promise.finally(() => { 1203 | console.log('SqlStoreIndex (' + this._rawTableName + '/' + this._indexTableName + ') getOnly: (' + 1204 | (Date.now() - startTime) + 'ms)'); 1205 | }); 1206 | } 1207 | return promise; 1208 | } 1209 | 1210 | getRange(keyLowRange: KeyType, keyHighRange: KeyType, lowRangeExclusive?: boolean, highRangeExclusive?: boolean, 1211 | reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number): Promise { 1212 | let checks: string; 1213 | let args: string[]; 1214 | const err = attempt(() => { 1215 | const ret = this._getRangeChecks(keyLowRange, keyHighRange, lowRangeExclusive, highRangeExclusive); 1216 | checks = ret.checks; 1217 | args = ret.args; 1218 | }); 1219 | if (err) { 1220 | return Promise.reject(err); 1221 | } 1222 | 1223 | let startTime: number; 1224 | if (this._verbose) { 1225 | startTime = Date.now(); 1226 | } 1227 | 1228 | let promise = this._handleQuery('SELECT nsp_data FROM ' + this._tableName + ' WHERE ' + checks!!!, args!!!, 1229 | reverseOrSortOrder, limit, offset); 1230 | if (this._verbose) { 1231 | promise = promise.finally(() => { 1232 | console.log('SqlStoreIndex (' + this._rawTableName + '/' + this._indexTableName + ') getRange: (' + 1233 | (Date.now() - startTime) + 'ms)'); 1234 | }); 1235 | } 1236 | return promise; 1237 | } 1238 | 1239 | // Warning: This function can throw, make sure to trap. 1240 | private _getRangeChecks(keyLowRange: KeyType, keyHighRange: KeyType, lowRangeExclusive?: boolean, 1241 | highRangeExclusive?: boolean) { 1242 | let checks: string[] = []; 1243 | let args: string[] = []; 1244 | if (keyLowRange !== null && keyLowRange !== undefined) { 1245 | checks.push(this._queryColumn + (lowRangeExclusive ? ' > ' : ' >= ') + '?'); 1246 | args.push(serializeKeyToString(keyLowRange, this._keyPath)); 1247 | } 1248 | if (keyHighRange !== null && keyHighRange !== undefined) { 1249 | checks.push(this._queryColumn + (highRangeExclusive ? ' < ' : ' <= ') + '?'); 1250 | args.push(serializeKeyToString(keyHighRange, this._keyPath)); 1251 | } 1252 | return { checks: checks.join(' AND '), args }; 1253 | } 1254 | 1255 | countAll(): Promise { 1256 | let startTime: number; 1257 | if (this._verbose) { 1258 | startTime = Date.now(); 1259 | } 1260 | 1261 | let promise = this._trans.runQuery('SELECT COUNT(*) cnt FROM ' + this._tableName).then(result => result[0]['cnt']); 1262 | if (this._verbose) { 1263 | promise = promise.finally(() => { 1264 | console.log('SqlStoreIndex (' + this._rawTableName + '/' + this._indexTableName + ') countAll: (' + 1265 | (Date.now() - startTime) + 'ms)'); 1266 | }); 1267 | } 1268 | return promise; 1269 | } 1270 | 1271 | countOnly(key: KeyType): Promise { 1272 | const joinedKey = attempt(() => { 1273 | return serializeKeyToString(key, this._keyPath); 1274 | }); 1275 | if (isError(joinedKey)) { 1276 | return Promise.reject(joinedKey); 1277 | } 1278 | 1279 | let startTime: number; 1280 | if (this._verbose) { 1281 | startTime = Date.now(); 1282 | } 1283 | 1284 | let promise = this._trans.runQuery('SELECT COUNT(*) cnt FROM ' + this._tableName + ' WHERE ' + this._queryColumn 1285 | + ' = ?', [joinedKey]).then(result => result[0]['cnt']); 1286 | if (this._verbose) { 1287 | promise = promise.finally(() => { 1288 | console.log('SqlStoreIndex (' + this._rawTableName + '/' + this._indexTableName + ') countOnly: (' + 1289 | (Date.now() - startTime) + 'ms)'); 1290 | }); 1291 | } 1292 | return promise; 1293 | } 1294 | 1295 | countRange(keyLowRange: KeyType, keyHighRange: KeyType, lowRangeExclusive?: boolean, highRangeExclusive?: boolean) 1296 | : Promise { 1297 | let checks: string; 1298 | let args: string[]; 1299 | const err = attempt(() => { 1300 | const ret = this._getRangeChecks(keyLowRange, keyHighRange, lowRangeExclusive, highRangeExclusive); 1301 | checks = ret.checks; 1302 | args = ret.args; 1303 | }); 1304 | if (err) { 1305 | return Promise.reject(err); 1306 | } 1307 | 1308 | let startTime: number; 1309 | if (this._verbose) { 1310 | startTime = Date.now(); 1311 | } 1312 | 1313 | let promise = this._trans.runQuery('SELECT COUNT(*) cnt FROM ' + this._tableName + ' WHERE ' + checks!!!, args!!!) 1314 | .then(result => result[0]['cnt']); 1315 | if (this._verbose) { 1316 | promise = promise.finally(() => { 1317 | console.log('SqlStoreIndex (' + this._rawTableName + '/' + this._indexTableName + ') countOnly: (' + 1318 | (Date.now() - startTime) + 'ms)'); 1319 | }); 1320 | } 1321 | return promise; 1322 | } 1323 | 1324 | fullTextSearch(searchPhrase: string, resolution: FullTextTermResolution = FullTextTermResolution.And, 1325 | limit?: number): Promise { 1326 | let startTime: number; 1327 | if (this._verbose) { 1328 | startTime = Date.now(); 1329 | } 1330 | 1331 | const terms = breakAndNormalizeSearchPhrase(searchPhrase); 1332 | if (terms.length === 0) { 1333 | return Promise.resolve([]); 1334 | } 1335 | 1336 | let promise: Promise; 1337 | if (this._supportsFTS3) { 1338 | if (resolution === FullTextTermResolution.And) { 1339 | promise = this._handleQuery('SELECT nsp_data FROM ' + this._tableName + ' WHERE ' + this._queryColumn + ' MATCH ?', 1340 | [map(terms, term => term + '*').join(' ')], false, limit); 1341 | } else if (resolution === FullTextTermResolution.Or) { 1342 | // SQLite FTS3 doesn't support OR queries so we have to hack it... 1343 | const baseQueries = map(terms, term => 'SELECT * FROM ' + this._indexTableName + ' WHERE nsp_key MATCH ?'); 1344 | const joinedQuery = 'SELECT * FROM (SELECT DISTINCT * FROM (' + baseQueries.join(' UNION ALL ') + ')) mi LEFT JOIN ' + 1345 | this._rawTableName + ' t ON mi.nsp_refpk = t.nsp_pk'; 1346 | const args = map(terms, term => term + '*'); 1347 | promise = this._handleQuery(joinedQuery, args, false, limit); 1348 | } else { 1349 | return Promise.reject('fullTextSearch called with invalid term resolution mode'); 1350 | } 1351 | } else { 1352 | let joinTerm: string; 1353 | if (resolution === FullTextTermResolution.And) { 1354 | joinTerm = ' AND '; 1355 | } else if (resolution === FullTextTermResolution.Or) { 1356 | joinTerm = ' OR '; 1357 | } else { 1358 | return Promise.reject('fullTextSearch called with invalid term resolution mode'); 1359 | } 1360 | 1361 | promise = this._handleQuery('SELECT nsp_data FROM ' + this._tableName + ' WHERE ' + 1362 | map(terms, term => this._queryColumn + ' LIKE ?').join(joinTerm), 1363 | map(terms, term => '%' + FakeFTSJoinToken + term + '%')); 1364 | } 1365 | if (this._verbose) { 1366 | promise = promise.finally(() => { 1367 | console.log('SqlStoreIndex (' + this._rawTableName + '/' + this._indexTableName + ') fullTextSearch: (' + 1368 | (Date.now() - startTime) + 'ms)'); 1369 | }); 1370 | } 1371 | return promise; 1372 | } 1373 | } 1374 | -------------------------------------------------------------------------------- /src/StoreHelpers.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * StoreHelpers.ts 3 | * Author: David de Regt 4 | * Copyright: Microsoft 2017 5 | * 6 | * Reusable helper classes for clients of NoSqlProvider to build more type-safe stores/indexes. 7 | */ 8 | 9 | import { DbIndex, QuerySortOrder, FullTextTermResolution, ItemType, KeyType, DbStore } from './NoSqlProvider'; 10 | 11 | export var ErrorCatcher: ((err: any) => Promise)|undefined = undefined; 12 | 13 | // Remove parens from full text search, crashes on React Native.... 14 | const FullTextSanitizeRegex = /[()]/g; 15 | 16 | // Encodes related type info into the Store/Index name. 17 | // The actual value is just a string, but the type system can extract this extra info. 18 | export type DBStore = string & { name?: Name, objectType?: ObjectType, keyFormat?: KeyFormat }; 19 | export type DBIndex, IndexKeyFormat> = string & { store?: Store, indexKeyFormat?: IndexKeyFormat }; 20 | 21 | export class SimpleTransactionIndexHelper { 22 | constructor(protected _index: DbIndex) { 23 | // Nothing to see here 24 | } 25 | 26 | getAll(reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number): Promise { 27 | let promise = this._index.getAll(reverseOrSortOrder, limit, offset) as Promise; 28 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 29 | } 30 | 31 | getOnly(key: IndexKeyFormat, reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number) 32 | : Promise { 33 | let promise = this._index.getOnly(key, reverseOrSortOrder, limit, offset) as Promise; 34 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 35 | } 36 | 37 | getRange(keyLowRange: IndexKeyFormat, keyHighRange: IndexKeyFormat, lowRangeExclusive?: boolean, highRangeExclusive?: boolean, 38 | reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number): Promise { 39 | let promise = this._index.getRange(keyLowRange, keyHighRange, lowRangeExclusive, 40 | highRangeExclusive, reverseOrSortOrder, limit, offset) as Promise; 41 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 42 | } 43 | 44 | countAll(): Promise { 45 | let promise = this._index.countAll(); 46 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 47 | } 48 | 49 | countOnly(key: IndexKeyFormat): Promise { 50 | let promise = this._index.countOnly(key); 51 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 52 | } 53 | 54 | countRange(keyLowRange: IndexKeyFormat, keyHighRange: IndexKeyFormat, 55 | lowRangeExclusive?: boolean, highRangeExclusive?: boolean): Promise { 56 | let promise = this._index.countRange(keyLowRange, keyHighRange, lowRangeExclusive, highRangeExclusive); 57 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 58 | } 59 | 60 | fullTextSearch(searchPhrase: string, resolution?: FullTextTermResolution, 61 | limit?: number): Promise { 62 | // Sanitize input by removing parens, the plugin on RN explodes 63 | let promise = this._index.fullTextSearch(searchPhrase.replace(FullTextSanitizeRegex, ''), 64 | resolution, limit) as Promise; 65 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 66 | } 67 | } 68 | 69 | export class SimpleTransactionStoreHelper { 70 | constructor(protected _store: DbStore, storeName /* Force type-checking */: DBStore) { 71 | // Nothing to see here 72 | } 73 | 74 | get(key: KeyFormat): Promise { 75 | let promise = this._store.get(key) as Promise; 76 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 77 | } 78 | 79 | getAll(sortOrder?: QuerySortOrder): Promise { 80 | let promise = this._store.openPrimaryKey().getAll(sortOrder) as Promise; 81 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 82 | } 83 | 84 | getOnly(key: KeyFormat, reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number) 85 | : Promise { 86 | let promise = this._store.openPrimaryKey().getOnly(key, reverseOrSortOrder, limit, offset) as Promise; 87 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 88 | } 89 | 90 | getRange(keyLowRange: KeyFormat, keyHighRange: KeyFormat, lowRangeExclusive?: boolean, highRangeExclusive?: boolean, 91 | reverseOrSortOrder?: boolean | QuerySortOrder, limit?: number, offset?: number): Promise { 92 | let promise = this._store.openPrimaryKey().getRange(keyLowRange, keyHighRange, 93 | lowRangeExclusive, highRangeExclusive, reverseOrSortOrder, limit, offset) as Promise; 94 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 95 | } 96 | 97 | getMultiple(keyOrKeys: KeyFormat|KeyFormat[]): Promise { 98 | let promise = this._store.getMultiple(keyOrKeys) as Promise; 99 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 100 | } 101 | 102 | openIndex(indexName: DBIndex, IndexKeyFormat>) 103 | : SimpleTransactionIndexHelper { 104 | return new SimpleTransactionIndexHelper(this._store.openIndex(indexName)); 105 | } 106 | 107 | openPrimaryKey(): SimpleTransactionIndexHelper { 108 | return new SimpleTransactionIndexHelper(this._store.openPrimaryKey()); 109 | } 110 | 111 | put(itemOrItems: ObjectType|ReadonlyArray): Promise { 112 | let promise = this._store.put(itemOrItems); 113 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 114 | } 115 | 116 | remove(keyOrKeys: KeyFormat|KeyFormat[]): Promise { 117 | let promise = this._store.remove(keyOrKeys); 118 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 119 | } 120 | 121 | clearAllData(): Promise { 122 | let promise = this._store.clearAllData(); 123 | return ErrorCatcher ? promise.catch(ErrorCatcher) : promise; 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/TransactionLockHelper.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * TransactionLockHelper.ts 3 | * Author: David de Regt 4 | * Copyright: Microsoft 2017 5 | * 6 | * Several of the different providers need various types of help enforcing exclusive/readonly transactions. This helper keeps 7 | * store-specific lock info and releases transactions at the right time, when the underlying provider can't handle it. 8 | */ 9 | 10 | import { ok } from 'assert'; 11 | import { map, some, find, Dictionary, findIndex } from 'lodash'; 12 | 13 | import { DbSchema } from './NoSqlProvider'; 14 | import { defer, IDeferred } from './defer'; 15 | 16 | export interface TransactionToken { 17 | readonly completionPromise: Promise; 18 | readonly storeNames: string[]; 19 | readonly exclusive: boolean; 20 | } 21 | 22 | interface PendingTransaction { 23 | token: TransactionToken; 24 | 25 | opened: boolean; 26 | openDefer: IDeferred; 27 | completionDefer: IDeferred|undefined; 28 | hadSuccess?: boolean; 29 | } 30 | 31 | export class TransactionLockHelper { 32 | private _closingDefer: IDeferred|undefined; 33 | private _closed = false; 34 | 35 | private _exclusiveLocks: Dictionary = {}; 36 | private _readOnlyCounts: Dictionary = {}; 37 | 38 | private _pendingTransactions: PendingTransaction[] = []; 39 | 40 | constructor(private _schema: DbSchema, private _supportsDiscreteTransactions: boolean) { 41 | for (const store of this._schema.stores) { 42 | this._exclusiveLocks[store.name] = false; 43 | this._readOnlyCounts[store.name] = 0; 44 | } 45 | } 46 | 47 | closeWhenPossible(): Promise { 48 | if (!this._closingDefer) { 49 | this._closingDefer = defer(); 50 | this._checkClose(); 51 | } 52 | 53 | return this._closingDefer.promise; 54 | } 55 | 56 | private _checkClose() { 57 | if (!this._closed && this._closingDefer && !this.hasTransaction() ) { 58 | this._closed = true; 59 | this._closingDefer.resolve(void 0); 60 | } 61 | } 62 | 63 | hasTransaction(): boolean { 64 | return this._pendingTransactions.length > 0 || 65 | some(this._exclusiveLocks, (value) => value) || 66 | some(this._readOnlyCounts, (value) => value > 0); 67 | } 68 | 69 | openTransaction(storeNames: string[]|undefined, exclusive: boolean): Promise { 70 | if (storeNames) { 71 | const missingStore = find(storeNames, name => !some(this._schema.stores, store => name === store.name)); 72 | if (missingStore) { 73 | return Promise.reject('Opened a transaction with a store name (' + missingStore + ') not defined in your schema!'); 74 | } 75 | } 76 | 77 | const completionDefer = defer(); 78 | const newToken: TransactionToken = { 79 | // Undefined means lock all stores 80 | storeNames: storeNames || map(this._schema.stores, store => store.name), 81 | exclusive, 82 | completionPromise: completionDefer.promise 83 | }; 84 | 85 | const pendingTrans: PendingTransaction = { 86 | token: newToken, 87 | opened: false, 88 | openDefer: defer(), 89 | completionDefer 90 | }; 91 | 92 | this._pendingTransactions.push(pendingTrans); 93 | 94 | this._checkNextTransactions(); 95 | 96 | return pendingTrans.openDefer.promise; 97 | } 98 | 99 | transactionComplete(token: TransactionToken) { 100 | const pendingTransIndex = findIndex(this._pendingTransactions, trans => trans.token === token); 101 | if (pendingTransIndex !== -1) { 102 | const pendingTrans = this._pendingTransactions[pendingTransIndex]; 103 | if (pendingTrans.completionDefer) { 104 | pendingTrans.hadSuccess = true; 105 | 106 | const toResolve = pendingTrans.completionDefer; 107 | this._pendingTransactions.splice(pendingTransIndex, 1); 108 | pendingTrans.completionDefer = undefined; 109 | toResolve.resolve(void 0); 110 | } else { 111 | throw new Error('Completing a transaction that has already been completed. Stores: ' + token.storeNames.join(',') + 112 | ', HadSuccess: ' + pendingTrans.hadSuccess); 113 | } 114 | } else { 115 | throw new Error('Completing a transaction that is no longer tracked. Stores: ' + token.storeNames.join(',')); 116 | } 117 | 118 | this._cleanTransaction(token); 119 | } 120 | 121 | transactionFailed(token: TransactionToken, message: string) { 122 | const pendingTransIndex = findIndex(this._pendingTransactions, trans => trans.token === token); 123 | if (pendingTransIndex !== -1) { 124 | const pendingTrans = this._pendingTransactions[pendingTransIndex]; 125 | if (pendingTrans.completionDefer) { 126 | pendingTrans.hadSuccess = false; 127 | 128 | const toResolve = pendingTrans.completionDefer; 129 | this._pendingTransactions.splice(pendingTransIndex, 1); 130 | pendingTrans.completionDefer = undefined; 131 | toResolve.reject(new Error(message)); 132 | } else { 133 | throw new Error('Failing a transaction that has already been completed. Stores: ' + token.storeNames.join(',') + 134 | ', HadSuccess: ' + pendingTrans.hadSuccess + ', Message: ' + message); 135 | } 136 | } else { 137 | throw new Error('Failing a transaction that is no longer tracked. Stores: ' + token.storeNames.join(',') + ', message: ' + 138 | message); 139 | } 140 | 141 | this._cleanTransaction(token); 142 | } 143 | 144 | private _cleanTransaction(token: TransactionToken) { 145 | if (token.exclusive) { 146 | for (const storeName of token.storeNames) { 147 | ok(this._exclusiveLocks[storeName], 'Missing expected exclusive lock for store: ' + storeName); 148 | this._exclusiveLocks[storeName] = false; 149 | } 150 | } else { 151 | for (const storeName of token.storeNames) { 152 | ok(this._readOnlyCounts[storeName] > 0, 'Missing expected readonly lock for store: ' + storeName); 153 | this._readOnlyCounts[storeName]--; 154 | } 155 | } 156 | 157 | this._checkNextTransactions(); 158 | } 159 | 160 | private _checkNextTransactions(): void { 161 | if (some(this._exclusiveLocks, lock => lock) && !this._supportsDiscreteTransactions) { 162 | // In these cases, no more transactions will be possible. Break out early. 163 | return; 164 | } 165 | 166 | for (let i = 0; i < this._pendingTransactions.length; ) { 167 | const trans = this._pendingTransactions[i]; 168 | 169 | if (trans.opened) { 170 | i++; 171 | continue; 172 | } 173 | 174 | if (this._closingDefer) { 175 | this._pendingTransactions.splice(i, 1); 176 | trans.openDefer.reject('Closing Provider'); 177 | continue; 178 | } 179 | 180 | if (some(trans.token.storeNames, storeName => this._exclusiveLocks[storeName] || 181 | (trans.token.exclusive && this._readOnlyCounts[storeName] > 0))) { 182 | i++; 183 | continue; 184 | } 185 | 186 | trans.opened = true; 187 | 188 | if (trans.token.exclusive) { 189 | for (const storeName of trans.token.storeNames) { 190 | this._exclusiveLocks[storeName] = true; 191 | } 192 | } else { 193 | for (const storeName of trans.token.storeNames) { 194 | this._readOnlyCounts[storeName]++; 195 | } 196 | } 197 | 198 | trans.openDefer.resolve(trans.token); 199 | } 200 | 201 | this._checkClose(); 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /src/WebSqlProvider.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * WebSqlProvider.ts 3 | * Author: David de Regt 4 | * Copyright: Microsoft 2015 5 | * 6 | * NoSqlProvider provider setup for WebSql, a browser storage backing. 7 | */ 8 | 9 | import { noop } from 'lodash'; 10 | import { IDeferred, defer } from './defer'; 11 | import { DbSchema } from './NoSqlProvider'; 12 | import { SQLDatabase, SqlProviderBase, SqlTransaction, SqliteSqlTransaction, SQLError, SQLTransaction } from './SqlProviderBase'; 13 | 14 | // Extending interfaces that should be in lib.d.ts but aren't for some reason. 15 | export interface SQLDatabaseCallback { 16 | (database: SQLDatabase): void; 17 | } 18 | 19 | declare global { 20 | interface Window { 21 | openDatabase(database_name: string, database_version: string, database_displayname: string, 22 | database_size?: number, creationCallback?: SQLDatabaseCallback): SQLDatabase; 23 | } 24 | } 25 | 26 | // The DbProvider implementation for WebSQL. This provider does a bunch of awkward stuff to pretend that a relational SQL store 27 | // is actually a NoSQL store. We store the raw object as a JSON.encoded string in the nsp_data column, and have an nsp_pk column 28 | // for the primary keypath value, then nsp_i_[index name] columns for each of the indexes. 29 | export class WebSqlProvider extends SqlProviderBase { 30 | private _db: SQLDatabase|undefined; 31 | 32 | constructor(supportsFTS3 = true) { 33 | super(supportsFTS3); 34 | } 35 | 36 | open(dbName: string, schema: DbSchema, wipeIfExists: boolean, verbose: boolean): Promise { 37 | super.open(dbName, schema, wipeIfExists, verbose); 38 | 39 | if (!window.openDatabase) { 40 | return Promise.reject('No support for WebSQL in this browser'); 41 | } 42 | 43 | try { 44 | this._db = window.openDatabase(dbName, '', dbName, 10 * 1024 * 1024); 45 | } catch (e) { 46 | if (e.code === 18) { 47 | // User rejected the quota attempt 48 | return Promise.reject('User rejected quota allowance'); 49 | } 50 | 51 | return Promise.reject('Unknown Exception opening WebSQL database: ' + e.toString()); 52 | } 53 | 54 | if (!this._db) { 55 | return Promise.reject('Couldn\'t open database: ' + dbName); 56 | } 57 | 58 | const upgradeDbDeferred = defer(); 59 | let changeVersionDeferred: IDeferred | undefined; 60 | let oldVersion = Number(this._db.version); 61 | if (oldVersion !== this._schema!!!.version) { 62 | // Needs a schema upgrade/change 63 | if (!wipeIfExists && this._schema!!!.version < oldVersion) { 64 | console.log('Database version too new (' + oldVersion + ') for schema version (' + this._schema!!!.version + '). Wiping!'); 65 | // Note: the reported DB version won't change back to the older number until after you do a put command onto the DB. 66 | wipeIfExists = true; 67 | } 68 | changeVersionDeferred = defer(); 69 | 70 | let errorDetail: string; 71 | this._db.changeVersion(this._db.version, this._schema!!!.version.toString(), (t) => { 72 | let trans = new WebSqlTransaction(t, defer().promise, this._schema!!!, this._verbose!!!, 999, 73 | this._supportsFTS3); 74 | 75 | this._upgradeDb(trans, oldVersion, wipeIfExists).then(() => { 76 | upgradeDbDeferred.resolve(void 0); 77 | }, err => { 78 | errorDetail = err && err.message ? err.message : err.toString(); 79 | // Got a promise error. Force the transaction to abort. 80 | trans.abort(); 81 | }); 82 | }, (err) => { 83 | upgradeDbDeferred.reject(err.message + (errorDetail ? ', Detail: ' + errorDetail : '')); 84 | }, () => { 85 | changeVersionDeferred!!!.resolve(void 0); 86 | } ); 87 | } else if (wipeIfExists) { 88 | // No version change, but wipe anyway 89 | let errorDetail: string; 90 | this.openTransaction([], true).then(trans => { 91 | this._upgradeDb(trans, oldVersion, true).then(() => { 92 | upgradeDbDeferred.resolve(void 0); 93 | }, err => { 94 | errorDetail = err && err.message ? err.message : err.toString(); 95 | // Got a promise error. Force the transaction to abort. 96 | trans.abort(); 97 | }); 98 | }, (err) => { 99 | upgradeDbDeferred.reject(err.message + (errorDetail ? ', Detail: ' + errorDetail : '')); 100 | }); 101 | } else { 102 | upgradeDbDeferred.resolve(void 0); 103 | } 104 | return upgradeDbDeferred.promise.then(() => changeVersionDeferred ? changeVersionDeferred.promise : undefined); 105 | } 106 | 107 | close(): Promise { 108 | this._db = undefined; 109 | return Promise.resolve(undefined); 110 | } 111 | 112 | protected _deleteDatabaseInternal(): Promise { 113 | return Promise.reject('No support for deleting'); 114 | } 115 | 116 | openTransaction(storeNames: string[], writeNeeded: boolean): Promise { 117 | if (!this._db) { 118 | return Promise.reject('Database closed'); 119 | } 120 | 121 | const deferred = defer(); 122 | 123 | let ourTrans: SqliteSqlTransaction|undefined; 124 | let finishDefer: IDeferred|undefined = defer(); 125 | (writeNeeded ? this._db.transaction : this._db.readTransaction).call(this._db, 126 | (trans: SQLTransaction) => { 127 | ourTrans = new WebSqlTransaction(trans, finishDefer!!!.promise, this._schema!!!, this._verbose!!!, 999, 128 | this._supportsFTS3); 129 | deferred.resolve(ourTrans); 130 | }, (err: SQLError) => { 131 | if (ourTrans) { 132 | // Got an error from inside the transaction. Error out all pending queries on the 133 | // transaction since they won't exit out gracefully for whatever reason. 134 | ourTrans.failAllPendingQueries(err); 135 | ourTrans.internal_markTransactionClosed(); 136 | if (finishDefer) { 137 | finishDefer.reject('WebSqlTransaction Error: ' + err.message); 138 | finishDefer = undefined; 139 | } 140 | } else { 141 | deferred.reject(err); 142 | } 143 | }, () => { 144 | ourTrans!!!.internal_markTransactionClosed(); 145 | if (finishDefer) { 146 | finishDefer.resolve(void 0); 147 | finishDefer = undefined; 148 | } 149 | }); 150 | 151 | return deferred.promise; 152 | } 153 | } 154 | 155 | class WebSqlTransaction extends SqliteSqlTransaction { 156 | constructor(protected trans: SQLTransaction, 157 | private _completionPromise: Promise, 158 | schema: DbSchema, 159 | verbose: boolean, 160 | maxVariables: number, 161 | supportsFTS3: boolean) { 162 | super(trans, schema, verbose, maxVariables, supportsFTS3); 163 | } 164 | 165 | getCompletionPromise(): Promise { 166 | return this._completionPromise; 167 | } 168 | 169 | abort(): void { 170 | // The only way to rollback a websql transaction is by forcing an error (which rolls back the trans): 171 | // http://stackoverflow.com/questions/16225320/websql-dont-rollback 172 | this.runQuery('ERROR ME TO DEATH').always(noop); 173 | } 174 | 175 | getErrorHandlerReturnValue(): boolean { 176 | // Causes a rollback on websql 177 | return true; 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /src/defer.ts: -------------------------------------------------------------------------------- 1 | export interface IDeferred { 2 | promise: Promise; 3 | resolve(value: T | PromiseLike): void; 4 | resolve(): void; 5 | reject(reason: unknown): void; 6 | } 7 | 8 | export function defer(): IDeferred { 9 | const deferred: Partial> = {}; 10 | // eslint-disable-next-line msteams/promise-must-complete 11 | deferred.promise = new Promise((resolve, reject) => { 12 | deferred.resolve = resolve; 13 | deferred.reject = reject; 14 | }); 15 | return deferred as IDeferred; 16 | } 17 | -------------------------------------------------------------------------------- /src/get-window.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Gets global window object - whether operating in worker or UI thread context. 3 | * Adapted from: https://stackoverflow.com/questions/7931182/reliably-detect-if-the-script-is-executing-in-a-web-worker 4 | */ 5 | export function getWindow() { 6 | if (typeof window === 'object' && window.document) { 7 | return window; 8 | } else if (self && self.document === undefined) { 9 | return self; 10 | } 11 | 12 | throw new Error('Undefined context'); 13 | } 14 | -------------------------------------------------------------------------------- /test.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Mocha Tests 5 | 6 | 7 | 8 |
9 | 10 | 11 | 12 | 13 | 14 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "declaration": true, 4 | "noResolve": false, 5 | "module": "commonjs", 6 | "target": "es5", 7 | "downlevelIteration": false, 8 | "importHelpers": true, 9 | "outDir": "dist/", 10 | "typeRoots": [ 11 | "node_modules/@types" 12 | ], 13 | "noImplicitAny": true, 14 | "noImplicitReturns": true, 15 | "noImplicitThis": true, 16 | "noUnusedLocals": true, 17 | "forceConsistentCasingInFileNames": true, 18 | "strict": true, 19 | "sourceMap": true 20 | }, 21 | 22 | "filesGlob": [ 23 | "src/**/*{ts,tsx}" 24 | ], 25 | 26 | "exclude": [ 27 | "dist", 28 | "node_modules" 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "align": [true, "statements"], 4 | "ban": [true, "_", {"name": ["_", "chain"]}], 5 | "class-name": true, 6 | "curly": true, 7 | "eofline": true, 8 | "forin": true, 9 | "indent": [true, "spaces"], 10 | "label-position": true, 11 | "max-line-length": [ true, 140 ], 12 | "no-arg": true, 13 | "no-bitwise": false, 14 | "no-consecutive-blank-lines": true, 15 | "no-string-throw": true, 16 | "no-construct": true, 17 | "no-debugger": true, 18 | "no-duplicate-variable": true, 19 | "no-empty": true, 20 | "no-eval": true, 21 | "no-switch-case-fall-through": true, 22 | "no-trailing-whitespace": false, 23 | "no-unused-expression": true, 24 | "one-line": [true, 25 | "check-open-brace", 26 | "check-catch", 27 | "check-else", 28 | "check-whitespace" 29 | ], 30 | "quotemark": [true, "single"], 31 | "radix": true, 32 | "semicolon": true, 33 | "triple-equals": [true, "allow-null-check"], 34 | "variable-name": false, 35 | "whitespace": [true, 36 | "check-branch", 37 | "check-decl", 38 | "check-operator", 39 | "check-separator", 40 | "check-type" 41 | ] 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | var webpack = require('webpack'); 3 | 4 | var webpackConfig = { 5 | entry: './src/tests/NoSqlProviderTests.ts', 6 | 7 | output: { 8 | filename: './NoSQLProviderTestsPack.js', 9 | }, 10 | 11 | externals: [ 'sqlite3', 'indexeddb-js', 'fs' ], 12 | 13 | resolve: { 14 | modules: [ 15 | path.resolve('./src'), 16 | path.resolve('./node_modules') 17 | ], 18 | extensions: ['.ts', '.tsx', '.js'] 19 | }, 20 | 21 | module: { 22 | rules: [{ 23 | // Compile TS. 24 | test: /\.tsx?$/, 25 | exclude: /node_modules/, 26 | loader: 'awesome-typescript-loader' 27 | }] 28 | }, 29 | 30 | mode: 'development' 31 | }; 32 | 33 | module.exports = webpackConfig; 34 | --------------------------------------------------------------------------------