├── TODO ├── test.html ├── README ├── testglue.js ├── test.js └── query.js /TODO: -------------------------------------------------------------------------------- 1 | TODO 2 | 3 | * error handling 4 | * limits 5 | * ordered results 6 | * test perf 7 | * investigate different strategies for filtering 8 | * investigate using (or at least supporting) workers for CPU-bound tasks 9 | * 'startswith' comparison for strings 10 | 11 | Separate/additional library: 12 | 13 | * indexing helpers (e.g. for full text searching, case folding, etc.) 14 | * query plan optimizer 15 | -------------------------------------------------------------------------------- /test.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | A small query library for IndexedDB 2 | 3 | 4 | Query construction 5 | ------------------ 6 | 7 | A query consists of the name of an index, an operation, and a 8 | comparison value. This is how you construct a query: 9 | 10 | Index("make").oneof("BMW", "Volkswagen") 11 | 12 | This will return all objects whose "make" index value is 13 | either "BMW" or "Volkswagen". Available operations are: 14 | 15 | * eq 16 | * lt, lteq 17 | * gt, gteq 18 | * between, betweeq 19 | * oneof 20 | 21 | It is possible to link queries with boolean operations, e.g.: 22 | 23 | Index("make").eq("BMW") 24 | .and(Index("model").eq("325i")) 25 | .and(Index("year").lteq(1991)) 26 | 27 | 28 | Getting results 29 | --------------- 30 | 31 | Getting results from a query works very much like getting results from 32 | a single index in IndexedDB. You have the option of a cursor, e.g.: 33 | 34 | let cars = []; 35 | let store = transaction.objectStore("cars"); 36 | let request = query.openCursor(store); 37 | request.onsuccess = function (event) { 38 | let cursor = request.result; 39 | if (cursor) { 40 | cars.push(cursor.value); 41 | cursor.continue(); 42 | } 43 | } 44 | 45 | or simply getting all values at once: 46 | 47 | let cars; 48 | let request = query.getAll(store); 49 | request.onsuccess = function (event) { 50 | cars = request.result; 51 | } 52 | 53 | `query.openKeyCursor` and `query.getAllKeys` are also available if 54 | just the keys are of interest. 55 | -------------------------------------------------------------------------------- /testglue.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Glue to make XPCShell test harness helpers work in a web page. 3 | */ 4 | function do_throw(text, stack) { 5 | console.error(text); 6 | throw text; 7 | } 8 | 9 | function print() { 10 | console.info.apply(console, arguments); 11 | } 12 | 13 | function do_execute_soon(func) { 14 | window.setTimeout(func, 0); 15 | } 16 | 17 | function do_test_pending() {} 18 | function do_test_finished() {} 19 | 20 | function do_check_eq(left, right) { 21 | let pass = left == right; 22 | console.log(pass ? "PASS" : "FAIL", left + " == " + right); 23 | if (!pass) { 24 | do_throw("FAIL"); 25 | } 26 | } 27 | 28 | function do_check_true(condition) { 29 | do_check_eq(condition, true); 30 | } 31 | 32 | const _TEST_FILE = "test.js"; 33 | 34 | 35 | /*** XPCShell test harness helpers ***/ 36 | 37 | /** 38 | * Add a test function to the list of tests that are to be run asynchronously. 39 | * 40 | * Each test function must call run_next_test() when it's done. Test files 41 | * should call run_next_test() in their run_test function to execute all 42 | * async tests. 43 | * 44 | * @return the test function that was passed in. 45 | */ 46 | let gTests = []; 47 | function add_test(func) { 48 | gTests.push(func); 49 | return func; 50 | } 51 | 52 | /** 53 | * Runs the next test function from the list of async tests. 54 | */ 55 | let gRunningTest = null; 56 | let gTestIndex = 0; // The index of the currently running test. 57 | function run_next_test() 58 | { 59 | function _run_next_test() 60 | { 61 | if (gTestIndex < gTests.length) { 62 | do_test_pending(); 63 | gRunningTest = gTests[gTestIndex++]; 64 | print("TEST-INFO | " + _TEST_FILE + " | Starting " + 65 | gRunningTest.name); 66 | // Exceptions do not kill asynchronous tests, so they'll time out. 67 | try { 68 | gRunningTest(); 69 | } 70 | catch (e) { 71 | do_throw(e); 72 | } 73 | } 74 | } 75 | 76 | // For sane stacks during failures, we execute this code soon, but not now. 77 | // We do this now, before we call do_test_finished(), to ensure the pending 78 | // counter (_tests_pending) never reaches 0 while we still have tests to run 79 | // (do_execute_soon bumps that counter). 80 | do_execute_soon(_run_next_test); 81 | 82 | if (gRunningTest !== null) { 83 | // Close the previous test do_test_pending call. 84 | do_test_finished(); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | /* Any copyright is dedicated to the Public Domain. 2 | http://creativecommons.org/publicdomain/zero/1.0/ */ 3 | 4 | function debug() { 5 | let args = Array.slice(arguments); 6 | args.unshift("DEBUG"); 7 | console.log.apply(console, args); 8 | } 9 | 10 | const DB_NAME = "testquery"; 11 | const DB_VERSION = 1; 12 | const STORE_NAME = "lemonscars"; 13 | 14 | let sampleRecords = [ 15 | {name: "ECTO-1", 16 | year: 1989, 17 | make: "BMW", 18 | model: "325i", 19 | races: 1}, 20 | {name: "ECTO-2", 21 | year: 1984, 22 | make: "BMW", 23 | model: "325e", 24 | races: 3}, 25 | {name: "Cheesy", 26 | year: 1984, 27 | make: "BMW", 28 | model: "325e", 29 | races: 9}, 30 | {name: "Pikachubaru", 31 | year: 2001, 32 | make: "Subaru", 33 | model: "Legacy Outback", 34 | races: 5}, 35 | {name: "Ferdinand the Bug", 36 | year: 1971, 37 | make: "Volkswagen", 38 | model: "Super Beetle", 39 | races: 0} 40 | ]; 41 | 42 | function openDB(callback) { 43 | let indexedDB = window.mozIndexedDB; 44 | let request = indexedDB.open(DB_NAME, DB_VERSION); 45 | request.onsuccess = function (event) { 46 | debug("Opened database:", DB_NAME, DB_VERSION); 47 | callback(request.result); 48 | }; 49 | request.onupgradeneeded = function (event) { 50 | debug("Database needs upgrade:", DB_NAME, 51 | event.oldVersion, event.newVersion); 52 | debug("Correct old database version:", event.oldVersion == 0); 53 | debug("Correct new database version:", event.newVersion == DB_VERSION); 54 | 55 | let db = request.result; 56 | let store = db.createObjectStore(STORE_NAME, {keyPath: "name"}); 57 | store.createIndex("year", "year", { unique: false }); 58 | store.createIndex("make", "make", { unique: false }); 59 | store.createIndex("model", "model", { unique: false }); 60 | store.createIndex("races", "races", { unique: false }); 61 | }; 62 | request.onerror = function (event) { 63 | debug("Failed to open database", DB_NAME); 64 | }; 65 | request.onblocked = function (event) { 66 | debug("Opening database request is blocked."); 67 | }; 68 | } 69 | 70 | let gDB; 71 | function populateDB(callback) { 72 | openDB(function (db) { 73 | gDB = db; 74 | let txn = gDB.transaction([STORE_NAME], IDBTransaction.READ_WRITE); 75 | let store = txn.objectStore(STORE_NAME); 76 | txn.oncomplete = function oncomplete() { 77 | console.debug("Populate transaction completed."); 78 | callback(); 79 | }; 80 | sampleRecords.forEach(function (record) { 81 | debug("Storing", record); 82 | store.put(record); 83 | }); 84 | }); 85 | } 86 | 87 | function openStore() { 88 | let txn = gDB.transaction([STORE_NAME], IDBTransaction.READ_ONLY); 89 | txn.oncomplete = run_next_test; 90 | txn.onabort = function () { 91 | console.error("The transaction was aborted because an error occurred."); 92 | }; 93 | let store = txn.objectStore(STORE_NAME); 94 | return store; 95 | } 96 | 97 | function compareKeys(keys, expectedKeys) { 98 | //TODO for now we don't care about order 99 | debug("Comparing", keys, expectedKeys); 100 | do_check_eq(keys.length, expectedKeys.length); 101 | do_check_eq(arrayUnion(keys, expectedKeys).length, keys.length); 102 | } 103 | 104 | function test(query, expectedKeys) { 105 | add_test(function test_openCursor() { 106 | debug("Testing " + query + ".openCursor"); 107 | let request = query.openCursor(openStore()); 108 | let keys = []; 109 | request.onsuccess = function onsuccess() { 110 | if (request.result == undefined) { 111 | compareKeys(keys, expectedKeys); 112 | return; 113 | } 114 | keys.push(request.result.name); 115 | }; 116 | }); 117 | 118 | add_test(function test_openKeyCursor() { 119 | debug("Testing " + query + ".openKeyCursor"); 120 | let request = query.openKeyCursor(openStore()); 121 | let keys = []; 122 | request.onsuccess = function onsuccess() { 123 | if (request.result == undefined) { 124 | compareKeys(keys, expectedKeys); 125 | return; 126 | } 127 | keys.push(request.result); 128 | }; 129 | }); 130 | 131 | add_test(function test_getAll() { 132 | debug("Testing " + query + ".getAll"); 133 | let request = query.getAll(openStore()); 134 | request.onsuccess = function onsuccess() { 135 | let keys = request.result.map(function (item) { return item.name; }); 136 | compareKeys(keys, expectedKeys); 137 | }; 138 | }); 139 | 140 | add_test(function test_getAllKeys() { 141 | debug("Testing " + query + ".getAllKeys"); 142 | let request = query.getAllKeys(openStore()); 143 | request.onsuccess = function onsuccess() { 144 | compareKeys(request.result, expectedKeys); 145 | }; 146 | }); 147 | 148 | } 149 | 150 | function run_tests() { 151 | populateDB(run_next_test); 152 | } 153 | 154 | 155 | /*** Tests start here ***/ 156 | 157 | // eq 158 | test( 159 | Index("make").eq("Chevrolet"), 160 | [] 161 | ); 162 | test( 163 | Index("make").eq("BMW"), 164 | ["ECTO-1", "ECTO-2", "Cheesy"] 165 | ); 166 | 167 | // neq 168 | test( 169 | Index("make").neq("BMW"), 170 | ["Pikachubaru", "Ferdinand the Bug"] 171 | ); 172 | 173 | // lt 174 | test( 175 | Index("year").lt(1971), //TODO 1971 fails on getAll?!? 176 | [] 177 | ); 178 | test( 179 | Index("year").lt(1984), //TODO 1984 fails on getAll?!? 180 | ["Ferdinand the Bug"] 181 | ); 182 | 183 | // lteq 184 | test( 185 | Index("year").lteq(1970), 186 | [] 187 | ); 188 | test( 189 | Index("year").lteq(1984), 190 | ["Ferdinand the Bug", "Cheesy", "ECTO-2"] 191 | ); 192 | 193 | // gt 194 | test( 195 | Index("year").gt(2001), 196 | [] 197 | ); 198 | test( 199 | Index("year").gt(1989), 200 | ["Pikachubaru"] 201 | ); 202 | 203 | // gteq 204 | test( 205 | Index("year").gteq(2002), 206 | [] 207 | ); 208 | test( 209 | Index("year").gteq(1989), 210 | ["Pikachubaru", "ECTO-1"] 211 | ); 212 | 213 | // oneof 214 | test( 215 | Index("make").oneof("Chevrolet", "Ford"), 216 | [] 217 | ); 218 | test( 219 | Index("make").oneof("Volkswagen", "Subaru"), 220 | ["Pikachubaru", "Ferdinand the Bug"] 221 | ); 222 | 223 | // between 224 | test( 225 | Index("year").between(1960, 1970), 226 | [] 227 | ); 228 | test( 229 | Index("year").between(1980, 1990), 230 | ["ECTO-1", "ECTO-2", "Cheesy"] 231 | ); 232 | 233 | 234 | // Composite queries 235 | test( 236 | Index("make").eq("BMW").and(Index("model").eq("325e")), 237 | ["ECTO-2", "Cheesy"] 238 | ); 239 | test( 240 | Index("make").eq("Volkswagen").or(Index("make").eq("Subaru")), 241 | ["Pikachubaru", "Ferdinand the Bug"] 242 | ); 243 | -------------------------------------------------------------------------------- /query.js: -------------------------------------------------------------------------------- 1 | /* This Source Code Form is subject to the terms of the Mozilla Public 2 | * License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | * You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 | 5 | /** 6 | * Represents a queriable index. Call methods on the returned object 7 | * to create queries. 8 | * 9 | * Example: 10 | * 11 | * let query = Index("make").eq("BMW"); 12 | * 13 | */ 14 | function Index(name) { 15 | function queryMaker(op) { 16 | return function () { 17 | return IndexQuery(name, op, arguments); 18 | }; 19 | } 20 | return { 21 | eq: queryMaker("eq"), 22 | neq: queryMaker("neq"), 23 | gt: queryMaker("gt"), 24 | gteq: queryMaker("gteq"), 25 | lt: queryMaker("lt"), 26 | lteq: queryMaker("lteq"), 27 | between: queryMaker("between"), 28 | betweeq: queryMaker("betweeq"), 29 | oneof: function oneof() { 30 | let values = Array.slice(arguments); 31 | let query = IndexQuery(name, "eq", [values.shift()]); 32 | while (values.length) { 33 | query = query.or(IndexQuery(name, "eq", [values.shift()])); 34 | } 35 | return query; 36 | } 37 | }; 38 | } 39 | 40 | /** 41 | * Helper that notifies a 'success' event on a request, with a given 42 | * result object. This is typically either a cursor or a result array. 43 | */ 44 | function notifySuccess(request, result) { 45 | let event = {type: "success", 46 | target: request}; //TODO complete event interface 47 | request.readyState = IDBRequest.DONE; 48 | request.result = result; 49 | if (typeof request.onsuccess == "function") { 50 | request.onsuccess(event); 51 | } 52 | }; 53 | 54 | /** 55 | * Create a cursor object. 56 | */ 57 | function Cursor(store, request, keys, keyOnly) { 58 | let cursor = { 59 | continue: function continue_() { 60 | if (!keys.length) { 61 | notifySuccess(request, undefined); 62 | return; 63 | } 64 | let key = keys.shift(); 65 | if (keyOnly) { 66 | cursor.key = key; 67 | notifySuccess(request, cursor); 68 | return; 69 | } 70 | let r = store.get(key); 71 | r.onsuccess = function onsuccess() { 72 | cursor.key = key; 73 | cursor.value = r.result; 74 | notifySuccess(request, cursor); 75 | }; 76 | } 77 | //TODO complete cursor interface 78 | }; 79 | return cursor; 80 | } 81 | 82 | /** 83 | * Create a request object. 84 | */ 85 | function Request() { 86 | return { 87 | result: undefined, 88 | onsuccess: null, 89 | onerror: null, 90 | readyState: IDBRequest.LOADING 91 | // TODO complete request interface 92 | }; 93 | } 94 | 95 | /** 96 | * Create a request that will receive a cursor. 97 | * 98 | * This will also kick off the query, instantiate the Cursor when the 99 | * results are available, and notify the first 'success' event. 100 | */ 101 | function CursorRequest(store, queryFunc, keyOnly) { 102 | let request = Request(); 103 | queryFunc(store, function (keys) { 104 | let cursor = Cursor(store, request, keys, keyOnly); 105 | cursor.continue(); 106 | }); 107 | return request; 108 | } 109 | 110 | /** 111 | * Create a request that will receive a result array. 112 | * 113 | * This will also kick off the query, build up the result array, and 114 | * notify the 'success' event. 115 | */ 116 | function ResultRequest(store, queryFunc, keyOnly) { 117 | let request = Request(); 118 | queryFunc(store, function (keys) { 119 | if (keyOnly || !keys.length) { 120 | notifySuccess(request, keys); 121 | return; 122 | } 123 | let results = []; 124 | function getNext() { 125 | let r = store.get(keys.shift()); 126 | r.onsuccess = function onsuccess() { 127 | results.push(r.result); 128 | if (!keys.length) { 129 | notifySuccess(request, results); 130 | return; 131 | } 132 | getNext(); 133 | }; 134 | } 135 | getNext(); 136 | }); 137 | return request; 138 | } 139 | 140 | /** 141 | * Provide a generic way to create a query object from a query function. 142 | * Depending on the implementation of that query function, the query could 143 | * produce results from an index, combine results from other queries, etc. 144 | */ 145 | function Query(queryFunc, toString) { 146 | 147 | let query = { 148 | 149 | // Sadly we need to expose this to make Intersection and Union work :( 150 | _queryFunc: queryFunc, 151 | 152 | and: function and(query2) { 153 | return Intersection(query, query2); 154 | }, 155 | 156 | or: function or(query2) { 157 | return Union(query, query2); 158 | }, 159 | 160 | openCursor: function openCursor(store) { 161 | return CursorRequest(store, queryFunc, false); 162 | }, 163 | 164 | openKeyCursor: function openKeyCursor(store) { 165 | return CursorRequest(store, queryFunc, true); 166 | }, 167 | 168 | getAll: function getAll(store) { 169 | return ResultRequest(store, queryFunc, false); 170 | }, 171 | 172 | getAllKeys: function getAllKeys(store) { 173 | return ResultRequest(store, queryFunc, true); 174 | }, 175 | 176 | toString: toString 177 | }; 178 | return query; 179 | }; 180 | 181 | /** 182 | * Create a query object that queries an index. 183 | */ 184 | function IndexQuery(indexName, operation, values) { 185 | let negate = false; 186 | let op = operation; 187 | if (op == "neq") { 188 | op = "eq"; 189 | negate = true; 190 | } 191 | 192 | function makeRange() { 193 | let range; 194 | switch (op) { 195 | case "eq": 196 | range = IDBKeyRange.only(values[0]); 197 | break; 198 | case "lt": 199 | range = IDBKeyRange.upperBound(values[0], true); 200 | break; 201 | case "lteq": 202 | range = IDBKeyRange.upperBound(values[0]); 203 | break; 204 | case "gt": 205 | range = IDBKeyRange.lowerBound(values[0], true); 206 | break; 207 | case "gteq": 208 | range = IDBKeyRange.lowerBound(values[0]); 209 | range.upperOpen = true; 210 | break; 211 | case "between": 212 | range = IDBKeyRange.bound(values[0], values[1], true, true); 213 | break; 214 | case "betweeq": 215 | range = IDBKeyRange.bound(values[0], values[1]); 216 | break; 217 | } 218 | return range; 219 | } 220 | 221 | function queryKeys(store, callback) { 222 | let index = store.index(indexName); 223 | let range = makeRange(); 224 | let request = index.getAllKeys(range); 225 | request.onsuccess = function onsuccess(event) { 226 | let result = request.result; 227 | if (!negate) { 228 | callback(result); 229 | return; 230 | } 231 | 232 | // Deal with the negation case. This means we fetch all keys and then 233 | // subtract the original result from it. 234 | request = index.getAllKeys(); 235 | request.onsuccess = function onsuccess(event) { 236 | let all = request.result; 237 | callback(arraySub(all, result)); 238 | }; 239 | }; 240 | } 241 | 242 | let args = arguments; 243 | function toString() { 244 | return "IndexQuery(" + Array.slice(args).toSource().slice(1, -1) + ")"; 245 | } 246 | 247 | return Query(queryKeys, toString); 248 | } 249 | 250 | /** 251 | * Create a query object that performs the intersection of two given queries. 252 | */ 253 | function Intersection(query1, query2) { 254 | function queryKeys(store, callback) { 255 | query1._queryFunc(store, function (keys1) { 256 | query2._queryFunc(store, function (keys2) { 257 | callback(arrayIntersect(keys1, keys2)); 258 | }); 259 | }); 260 | } 261 | 262 | function toString() { 263 | return "Intersection(" + query1.toString() + ", " + query2.toString() + ")"; 264 | } 265 | 266 | return Query(queryKeys, toString); 267 | } 268 | 269 | /** 270 | * Create a query object that performs the union of two given queries. 271 | */ 272 | function Union(query1, query2) { 273 | function queryKeys(store, callback) { 274 | query1._queryFunc(store, function (keys1) { 275 | query2._queryFunc(store, function (keys2) { 276 | callback(arrayUnion(keys1, keys2)); 277 | }); 278 | }); 279 | } 280 | 281 | function toString() { 282 | return "Union(" + query1.toString() + ", " + query2.toString() + ")"; 283 | } 284 | 285 | return Query(queryKeys, toString); 286 | } 287 | 288 | 289 | function arraySub(minuend, subtrahend) { 290 | if (!minuend.length || !subtrahend.length) { 291 | return minuend; 292 | } 293 | return minuend.filter(function(item) { 294 | return subtrahend.indexOf(item) == -1; 295 | }); 296 | } 297 | 298 | function arrayUnion(foo, bar) { 299 | if (!foo.length) { 300 | return bar; 301 | } 302 | if (!bar.length) { 303 | return foo; 304 | } 305 | return foo.concat(arraySub(bar, foo)); 306 | } 307 | 308 | function arrayIntersect(foo, bar) { 309 | if (!foo.length) { 310 | return foo; 311 | } 312 | if (!bar.length) { 313 | return bar; 314 | } 315 | return foo.filter(function(item) { 316 | return bar.indexOf(item) != -1; 317 | }); 318 | } 319 | --------------------------------------------------------------------------------