├── .gitignore ├── LICENSE ├── README.md ├── include └── sqlite_protobuf.h ├── meson.build ├── proto_table ├── proto_table.c └── proto_table.h └── src ├── extension_main.cpp ├── protobuf_enum.cpp ├── protobuf_enum.h ├── protobuf_extract.cpp ├── protobuf_extract.h ├── protobuf_json.cpp ├── protobuf_json.h ├── protobuf_load.cpp ├── protobuf_load.h ├── protobuf_text.cpp ├── protobuf_text.h ├── sqlite3.h ├── sqlite3ext.h ├── utilities.cpp └── utilities.h /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *# 3 | *.swp 4 | *.swo 5 | *.o 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Backtrace I/O, Inc 4 | Copyright (c) 2019 Ryan Govostes 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining 7 | a copy of this software and associated documentation files (the 8 | "Software"), to deal in the Software without restriction, including 9 | without limitation the rights to use, copy, modify, merge, publish, 10 | distribute, sublicense, and/or sell copies of the Software, and to 11 | permit persons to whom the Software is furnished to do so, subject to 12 | the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be 15 | included in all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This fork of [sqlite\_protobuf](https://github.com/rgov/sqlite_protobuf) 2 | fixes some issues (e.g., 3 | [#15](https://github.com/rgov/sqlite_protobuf/issues/15)) and removes the 4 | test suite that we do not use. 5 | 6 | It also comes with `proto_table`, a C library to construct ergonomic 7 | views on top of the `sqlite_protobuf` extension. 8 | 9 | # Protobuf Extension for SQLite 10 | 11 | This project implements a [run-time loadable extension][ext] for 12 | [SQLite][]. It allows SQLite to perform queries that can extract field 13 | values out of stored Protobuf messages. 14 | 15 | [ext]: https://www.sqlite.org/loadext.html 16 | [SQLite]: https://www.sqlite.org/ 17 | 18 | See also the [JSON1 Extension][json1], which is a similar extension 19 | for querying JSON-encoded database content. 20 | 21 | [json1]: https://www.sqlite.org/json1.html 22 | -------------------------------------------------------------------------------- /include/sqlite_protobuf.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | struct sqlite3; 4 | struct sqlite3_api_routines; 5 | 6 | #ifdef __cplusplus 7 | namespace sqlite_protobuf { 8 | extern "C" { 9 | #endif 10 | 11 | int sqlite3_sqliteprotobuf_init(struct sqlite3 *db, char **pzErrMsg, 12 | const struct sqlite3_api_routines *pApi); 13 | 14 | #ifdef __cplusplus 15 | } 16 | } // namespace sqlite_protobuf 17 | #endif 18 | -------------------------------------------------------------------------------- /meson.build: -------------------------------------------------------------------------------- 1 | sqlite_protobuf_src_dir = 'src' 2 | sqlite_protobuf_include_dir = include_directories('include') 3 | 4 | sqlite_protobuf_src_files = ''' 5 | extension_main.cpp 6 | protobuf_enum.cpp 7 | protobuf_extract.cpp 8 | protobuf_load.cpp 9 | protobuf_json.cpp 10 | protobuf_text.cpp 11 | utilities.cpp 12 | '''.split() 13 | 14 | _sqlite_protobuf_src_files = [] 15 | foreach s : sqlite_protobuf_src_files 16 | _sqlite_protobuf_src_files += join_paths(sqlite_protobuf_src_dir, s) 17 | endforeach 18 | 19 | sqlite_protobuf_lib = static_library('sqlite_protobuf', 20 | _sqlite_protobuf_src_files, 21 | include_directories: sqlite_protobuf_include_dir, 22 | dependencies: [libprotobuf_dep, libdl_dep], 23 | cpp_args: ['-Wno-undef']) 24 | 25 | libsqlite_protobuf_dep = declare_dependency(link_whole: sqlite_protobuf_lib, 26 | include_directories: sqlite_protobuf_include_dir) 27 | 28 | sqlite_protobuf_so = shared_library('sqlite_protobuf', 29 | _sqlite_protobuf_src_files, 30 | include_directories: sqlite_protobuf_include_dir, 31 | dependencies: [libprotobuf_dep, libdl_dep], 32 | cpp_args: ['-Wno-undef'], 33 | install: true) 34 | -------------------------------------------------------------------------------- /proto_table/proto_table.c: -------------------------------------------------------------------------------- 1 | #include "proto_table.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | /* 11 | * Try to autocommit every `AUTOCOMMIT_BATCH_SIZE` write operation by 12 | * default. 13 | * 14 | * This value should be high enough to amortise the commit overhead 15 | * (fsync), but not so high that the write journal grows too large for 16 | * the page cache. In practice, this means we should make it as small 17 | * as possible while preserving decent performance. 18 | */ 19 | #define AUTOCOMMIT_BATCH_SIZE 20000 20 | 21 | static struct umash_params index_fp_params; 22 | 23 | extern int proto_prepare(sqlite3 *, sqlite3_stmt **, const char *); 24 | 25 | /** 26 | * We store the expression that corresponds to each of the view's 27 | * column in an array of these key-value structs. 28 | */ 29 | struct view_column { 30 | const char *column_name; 31 | char *expression; 32 | bool auto_index; 33 | }; 34 | 35 | __attribute__((__constructor__)) static void 36 | populate_fp_params(void) 37 | { 38 | static const char key[32] = "proto table umash index fp key"; 39 | 40 | umash_params_derive(&index_fp_params, 0, key); 41 | return; 42 | } 43 | 44 | /** 45 | * Generates a statement that will create an index on demand for 46 | * `index`, given the view column definitions in `columns`. 47 | */ 48 | static char * 49 | create_index(char **OUT_index_name, const char *table_name, 50 | const struct view_column *columns, size_t num_columns, 51 | const struct proto_index *index, bool auto_index) 52 | { 53 | char *index_name = NULL; 54 | char *index_expr; 55 | char *ret; 56 | 57 | *OUT_index_name = NULL; 58 | 59 | /* Construct the index expression. */ 60 | index_expr = strdup(""); 61 | if (index_expr == NULL) 62 | goto fail; 63 | 64 | for (size_t i = 0; index->components[i] != NULL; i++) { 65 | const char *component = index->components[i]; 66 | const char *prefix = (i == 0) ? "\n " : ",\n "; 67 | char *update; 68 | 69 | /* 70 | * See if we want to replace that with the view 71 | * column's expansion. 72 | */ 73 | for (size_t j = 0; j < num_columns; j++) { 74 | if (strcmp(component, columns[j].column_name) == 0) { 75 | component = columns[j].expression; 76 | break; 77 | } 78 | } 79 | 80 | if (asprintf(&update, "%s%s%s", index_expr, prefix, component) < 0) 81 | goto fail; 82 | 83 | free(index_expr); 84 | index_expr = update; 85 | } 86 | 87 | /* Compute the index's name based on the index expression. */ 88 | { 89 | struct umash_fp fp; 90 | 91 | fp = umash_fprint( 92 | &index_fp_params, 0, index_expr, strlen(index_expr)); 93 | if (asprintf(&index_name, 94 | "proto_%sindex__%s__%s__%016" PRIx64 "%016" PRIx64, 95 | (auto_index ? "auto" : ""), table_name, index->name_suffix, 96 | fp.hash[0], fp.hash[1]) < 0) { 97 | index_name = NULL; 98 | goto fail; 99 | } 100 | } 101 | 102 | /* 103 | * Re-use the old index if it already exists: we don't want to 104 | * recreate it. 105 | */ 106 | if (asprintf(&ret, 107 | "CREATE INDEX IF NOT EXISTS %s\n" 108 | "ON %s_raw(%s\n);", 109 | index_name, table_name, index_expr) < 0) 110 | goto fail; 111 | 112 | free(index_expr); 113 | *OUT_index_name = index_name; 114 | return ret; 115 | 116 | fail: 117 | free(index_name); 118 | free(index_expr); 119 | return NULL; 120 | } 121 | 122 | /** 123 | * Returns a series of SQL statements to create the proto table's 124 | * backing raw table, as well as the view table, triggers, and 125 | * indexes on protopaths. 126 | * 127 | * The last statement is a `SELECT` statement that will list the name 128 | * of all indexes attached to the raw table that aren't necessary 129 | * anymore. 130 | */ 131 | static char * 132 | generate_proto_table(const struct proto_table *table) 133 | { 134 | char *create_raw = NULL; 135 | char *create_view = NULL; 136 | char *create_triggers = NULL; 137 | /* A list of column names, with a comma before each one. */ 138 | char *column_names = NULL; 139 | /* 140 | * A list of the corresponding CAST expressions, with a comma 141 | * before each one. 142 | */ 143 | char *column_expressions = NULL; 144 | /* 145 | * A comma-separated list of the index we want to create on 146 | * the "raw" table. 147 | */ 148 | char *index_names = NULL; 149 | char *create_indexes = NULL; 150 | char *select_bad_indexes = NULL; 151 | struct view_column *view_columns = NULL; 152 | size_t num_view_columns = 0; 153 | char *ret = NULL; 154 | 155 | /* 156 | * Make sure the raw table exists. No-op if there's already a 157 | * raw table: we don't want to drop all that data. 158 | */ 159 | if (asprintf(&create_raw, 160 | "CREATE TABLE IF NOT EXISTS %s_raw (\n" 161 | " id INTEGER PRIMARY KEY ASC NOT NULL,\n" 162 | " proto BLOB NOT NULL" 163 | ");", 164 | table->name) < 0) { 165 | create_raw = NULL; 166 | goto fail; 167 | } 168 | 169 | for (num_view_columns = 0; 170 | table->columns != NULL && table->columns[num_view_columns].name != NULL; 171 | num_view_columns++) 172 | ; 173 | 174 | view_columns = calloc(num_view_columns, sizeof(view_columns[0])); 175 | column_names = strdup(""); 176 | column_expressions = strdup(""); 177 | 178 | if (column_names == NULL || column_expressions == NULL) 179 | goto fail; 180 | 181 | for (size_t i = 0; i < num_view_columns; i++) { 182 | const struct proto_column *column = &table->columns[i]; 183 | struct view_column *view = &view_columns[i]; 184 | char *update; 185 | 186 | view->column_name = column->name; 187 | if (asprintf(&view->expression, 188 | "CAST(protobuf_extract(proto, '%s', '%s', NULL) AS %s)", 189 | table->message_name, column->path, column->type) < 0) { 190 | view->expression = NULL; 191 | goto fail; 192 | } 193 | 194 | /* Weak selectors don't get auto indexes. */ 195 | switch (column->index) { 196 | case PROTO_SELECTOR_TYPE_STRONG: 197 | view->auto_index = true; 198 | break; 199 | 200 | case PROTO_SELECTOR_TYPE_WEAK: 201 | view->auto_index = false; 202 | break; 203 | 204 | default: 205 | /* Index when in doubt. */ 206 | view->auto_index = true; 207 | break; 208 | } 209 | 210 | /* 211 | * The simple string concatenation here is 212 | * quadratic-time and involves a lot of heap allocation, 213 | * but you have bigger problems with your schemas if 214 | * that's an issue. 215 | */ 216 | if (asprintf(&update, "%s,\n %s", column_names, column->name) < 0) 217 | goto fail; 218 | free(column_names); 219 | column_names = update; 220 | 221 | if (asprintf(&update, "%s,\n %s", column_expressions, 222 | view->expression) < 0) 223 | goto fail; 224 | free(column_expressions); 225 | column_expressions = update; 226 | } 227 | 228 | /* 229 | * Re-recreate our view: it's ok to drop the old view if any, 230 | * since it doesn't hold any data. 231 | */ 232 | if (asprintf(&create_view, 233 | "DROP VIEW IF EXISTS %s;\n" 234 | "CREATE VIEW %s (\n" 235 | " id,\n" 236 | " proto%s\n" 237 | ") AS SELECT\n" 238 | " id,\n" 239 | " proto%s\n" 240 | "FROM %s_raw;", 241 | table->name, table->name, column_names, column_expressions, 242 | table->name) < 0) { 243 | create_view = NULL; 244 | goto fail; 245 | } 246 | 247 | /* 248 | * Same thing for the triggers that map mutations on the view 249 | * to mutation of the underlying raw table. 250 | */ 251 | if (asprintf(&create_triggers, 252 | "DROP TRIGGER IF EXISTS %1$s_insert;\n" 253 | "CREATE TRIGGER %1$s_insert INSTEAD OF INSERT ON %1$s\n" 254 | "BEGIN\n" 255 | " INSERT INTO %1$s_raw(proto) VALUES(NEW.proto);\n" 256 | "END;\n" 257 | "DROP TRIGGER IF EXISTS %1$s_update;\n" 258 | "CREATE TRIGGER %1$s_update INSTEAD OF UPDATE OF proto ON %1$s\n" 259 | "BEGIN\n" 260 | " UPDATE %1$s_raw SET proto = NEW.proto WHERE id = OLD.id;\n" 261 | "END;\n" 262 | "DROP TRIGGER IF EXISTS %1$s_delete;\n" 263 | "CREATE TRIGGER %1$s_delete INSTEAD OF DELETE ON %1$s\n" 264 | "BEGIN\n" 265 | " DELETE FROM %1$s_raw WHERE id = OLD.id;\n" 266 | "END;", 267 | table->name) < 0) { 268 | create_triggers = NULL; 269 | goto fail; 270 | } 271 | 272 | /* Add an index for each view column. */ 273 | create_indexes = strdup(""); 274 | for (size_t i = 0; i < num_view_columns; i++) { 275 | const struct proto_index index = { 276 | .name_suffix = view_columns[i].column_name, 277 | .components = 278 | (const char *[]) { 279 | view_columns[i].column_name, 280 | NULL, 281 | }, 282 | }; 283 | char *index_name, *stmt, *update; 284 | int r; 285 | 286 | if (view_columns[i].auto_index == false) 287 | continue; 288 | 289 | stmt = create_index(&index_name, table->name, view_columns, 290 | num_view_columns, &index, /*auto_index=*/true); 291 | if (stmt == NULL) 292 | goto fail; 293 | 294 | #define HANDLE_INDEX() \ 295 | do { \ 296 | if (index_names == NULL) { \ 297 | r = asprintf(&update, "'%s'", index_name); \ 298 | } else { \ 299 | r = asprintf(&update, "%s, '%s'", index_names, index_name); \ 300 | } \ 301 | \ 302 | free(index_name); \ 303 | if (r < 0) { \ 304 | free(stmt); \ 305 | goto fail; \ 306 | } \ 307 | \ 308 | free(index_names); \ 309 | index_names = update; \ 310 | \ 311 | r = asprintf(&update, "%s\n%s", create_indexes, stmt); \ 312 | free(stmt); \ 313 | if (r < 0) \ 314 | goto fail; \ 315 | \ 316 | free(create_indexes); \ 317 | create_indexes = update; \ 318 | } while (0) 319 | 320 | HANDLE_INDEX(); 321 | } 322 | 323 | /* And now add any additional index. */ 324 | for (size_t i = 0; 325 | table->indexes != NULL && table->indexes[i].name_suffix != NULL; i++) { 326 | char *index_name, *stmt, *update; 327 | int r; 328 | 329 | stmt = create_index(&index_name, table->name, view_columns, 330 | num_view_columns, &table->indexes[i], /*auto_index=*/false); 331 | if (stmt == NULL) 332 | goto fail; 333 | 334 | HANDLE_INDEX(); 335 | 336 | #undef HANDLE_INDEX 337 | } 338 | 339 | /* 340 | * List all `proto_index__` and `proto_autoindex__` indexes 341 | * associated with the underlying raw table that we wouldn't 342 | * have generated ourselves: we have to get rid of them, in 343 | * case they refer to now-unknown fields in the protobuf 344 | * message. 345 | */ 346 | if (asprintf(&select_bad_indexes, 347 | "SELECT name FROM sqlite_master WHERE\n" 348 | " type = 'index' AND tbl_name = '%s_raw' AND\n" 349 | " (name LIKE 'proto_index__%%' OR name LIKE 'proto_autoindex__%%') AND\n" 350 | " name NOT IN (%s);", 351 | table->name, index_names) < 0) { 352 | select_bad_indexes = NULL; 353 | goto fail; 354 | } 355 | 356 | if (asprintf(&ret, 357 | "BEGIN EXCLUSIVE TRANSACTION;\n" 358 | "%s\n%s\n\n%s\n%s\n" 359 | "COMMIT TRANSACTION;\n" 360 | "\n%s", 361 | create_raw, create_view, create_triggers, create_indexes, 362 | select_bad_indexes) < 0) { 363 | ret = NULL; 364 | goto fail; 365 | } 366 | 367 | out: 368 | free(create_raw); 369 | free(create_view); 370 | free(create_triggers); 371 | free(column_names); 372 | free(column_expressions); 373 | free(index_names); 374 | free(create_indexes); 375 | 376 | for (size_t i = 0; i < num_view_columns; i++) { 377 | free(view_columns[i].expression); 378 | } 379 | 380 | free(view_columns); 381 | free(select_bad_indexes); 382 | return ret; 383 | 384 | fail: 385 | free(ret); 386 | ret = NULL; 387 | goto out; 388 | } 389 | 390 | /** 391 | * We accumulate the name of indexes we want to get rid of in this 392 | * struct. 393 | */ 394 | struct bad_indexes { 395 | char **names; 396 | size_t num_names; 397 | bool log_to_stderr; 398 | }; 399 | 400 | static int 401 | bad_indexes_callback(void *thunk, int nargs, char **values, char **columns) 402 | { 403 | struct bad_indexes *bad_indexes = thunk; 404 | char **new_names; 405 | char *name; 406 | 407 | (void)columns; 408 | if (nargs != 1) 409 | return 0; 410 | 411 | if (bad_indexes->log_to_stderr == true) 412 | fprintf(stderr, "Found unwanted proto_index: %s\n", values[0]); 413 | 414 | name = strdup(values[0]); 415 | if (name == NULL) 416 | return -1; 417 | 418 | new_names = realloc(bad_indexes->names, 419 | (bad_indexes->num_names + 1) * sizeof(bad_indexes->names[0])); 420 | if (new_names == NULL) { 421 | free(name); 422 | return -1; 423 | } 424 | 425 | bad_indexes->names = new_names; 426 | bad_indexes->names[bad_indexes->num_names++] = name; 427 | return 0; 428 | } 429 | 430 | /** 431 | * Ensures the `spec`ced table in `db` is in the expected state. 432 | * 433 | * If the `command_cache` pointee is NULL, the pointer will be 434 | * populated with a cached SQL string that corresponds to `spec`; if 435 | * its pointee is non-NULL, it must have been generated by a prior 436 | * call to `setup_proto_table` for the same `spec`. 437 | */ 438 | int 439 | proto_table_setup(char **command_cache, sqlite3 *db, const struct proto_table *spec) 440 | { 441 | struct bad_indexes bad_indexes = { 442 | .log_to_stderr = spec->log_sql_to_stderr, 443 | }; 444 | char *error; 445 | int rc; 446 | 447 | if (*command_cache == NULL) { 448 | *command_cache = generate_proto_table(spec); 449 | if (*command_cache == NULL) 450 | return SQLITE_NOMEM; 451 | 452 | if (spec->log_sql_to_stderr == true) { 453 | fprintf(stderr, "proto_index SQL for %s:\n%s\n", spec->name, 454 | *command_cache); 455 | } 456 | } 457 | 458 | rc = sqlite3_exec( 459 | db, *command_cache, bad_indexes_callback, &bad_indexes, &error); 460 | if (rc != SQLITE_OK) 461 | goto fail_sqlite; 462 | 463 | for (size_t i = 0; i < bad_indexes.num_names; i++) { 464 | char *remove_bad_index; 465 | 466 | if (asprintf(&remove_bad_index, "DROP INDEX IF EXISTS \"%s\";", 467 | bad_indexes.names[i]) < 0) { 468 | rc = SQLITE_NOMEM; 469 | goto out; 470 | } 471 | 472 | rc = sqlite3_exec(db, remove_bad_index, NULL, NULL, &error); 473 | free(remove_bad_index); 474 | if (rc != SQLITE_OK) 475 | goto fail_sqlite; 476 | } 477 | 478 | out: 479 | for (size_t i = 0; i < bad_indexes.num_names; i++) 480 | free(bad_indexes.names[i]); 481 | free(bad_indexes.names); 482 | return rc; 483 | 484 | fail_sqlite: 485 | fprintf(stderr, "sqlite_exec failed for table %s: %s, rc=%i\n", spec->name, 486 | error ?: "unknown error", rc); 487 | sqlite3_free(error); 488 | goto out; 489 | } 490 | 491 | int64_t 492 | proto_table_paginate(sqlite3 *db, const char *table, int64_t begin, size_t wanted) 493 | { 494 | char *template; 495 | sqlite3_stmt *stmt = NULL; 496 | int64_t ret; 497 | int rc; 498 | 499 | if (asprintf(&template, 500 | " SELECT COALESCE(MAX(id), :begin)" 501 | " FROM (" 502 | " SELECT id" 503 | " FROM `%s`" 504 | " WHERE id > :begin" 505 | " ORDER BY id ASC" 506 | " LIMIT :wanted" 507 | " )", 508 | table) < 0) { 509 | return -(int64_t)SQLITE_NOMEM; 510 | } 511 | 512 | rc = proto_prepare(db, &stmt, template); 513 | if (rc != SQLITE_OK) { 514 | fprintf( 515 | stderr, "failed to prepare pagination statement. rc=%i\n", rc); 516 | ret = -(int64_t)rc; 517 | goto out; 518 | } 519 | 520 | if ((rc = PROTO_BIND(stmt, ":begin", begin)) != SQLITE_OK || 521 | (rc = PROTO_BIND(stmt, ":wanted", wanted)) != SQLITE_OK) { 522 | fprintf(stderr, "failed to bind pagination parameters. rc=%i\n", rc); 523 | ret = -(int64_t)rc; 524 | goto out; 525 | } 526 | 527 | rc = sqlite3_step(stmt); 528 | switch (rc) { 529 | case SQLITE_DONE: 530 | ret = begin; 531 | break; 532 | 533 | case SQLITE_ROW: 534 | ret = sqlite3_column_int64(stmt, 0); 535 | break; 536 | 537 | default: 538 | fprintf(stderr, "failed to execute pagination query. rc=%i\n", rc); 539 | ret = -(int64_t)rc; 540 | break; 541 | } 542 | 543 | out: 544 | (void)sqlite3_finalize(stmt); 545 | free(template); 546 | return ret; 547 | } 548 | 549 | int 550 | proto_db_transaction_begin(struct proto_db *db) 551 | { 552 | char *err = NULL; 553 | int rc; 554 | 555 | if (db->transaction_depth++ > 0) { 556 | assert(db->transaction_depth > 0); 557 | return SQLITE_OK; 558 | } 559 | 560 | rc = sqlite3_exec(db->db, "BEGIN IMMEDIATE TRANSACTION;", NULL, NULL, &err); 561 | if (rc != 0) { 562 | db->transaction_depth--; 563 | fprintf(stderr, "failed to open sqlite transaction, rc=%i: %s\n", rc, 564 | err ?: "unknown error"); 565 | } 566 | 567 | sqlite3_free(err); 568 | return rc; 569 | } 570 | 571 | void 572 | proto_db_transaction_end(struct proto_db *db) 573 | { 574 | char *err = NULL; 575 | int rc; 576 | 577 | assert(db->transaction_depth > 0); 578 | if (--db->transaction_depth > 0) { 579 | /* Cycle if we now can. */ 580 | proto_db_count_writes(db, 0); 581 | return; 582 | } 583 | 584 | db->write_count = 0; 585 | rc = sqlite3_exec(db->db, "COMMIT TRANSACTION;", NULL, NULL, &err); 586 | if (rc != 0) { 587 | fprintf(stderr, "failed to commit sqlite transaction, rc=%i: %s\n", 588 | rc, err ?: "unknown error"); 589 | abort(); 590 | } 591 | 592 | sqlite3_free(err); 593 | return; 594 | } 595 | 596 | int 597 | proto_db_batch_begin(struct proto_db *db) 598 | { 599 | 600 | db->autocommit_depth++; 601 | assert(db->autocommit_depth > 0); 602 | return proto_db_transaction_begin(db); 603 | } 604 | 605 | void 606 | proto_db_batch_end(struct proto_db *db) 607 | { 608 | 609 | proto_db_transaction_end(db); 610 | assert(db->autocommit_depth > 0); 611 | db->autocommit_depth--; 612 | return; 613 | } 614 | 615 | void 616 | proto_db_count_writes(struct proto_db *db, size_t n) 617 | { 618 | char *err = NULL; 619 | uint32_t batch_size = db->batch_size ?: AUTOCOMMIT_BATCH_SIZE; 620 | int rc; 621 | 622 | if (db->transaction_depth == 0) 623 | return; 624 | 625 | if (db->write_count < batch_size && n < batch_size - db->write_count) { 626 | db->write_count += n; 627 | return; 628 | } 629 | 630 | db->write_count = batch_size; 631 | if (db->autocommit_depth < db->transaction_depth) 632 | return; 633 | 634 | /* 635 | * We want to and can flush writes. Close the current 636 | * transaction and immediately open a new one. 637 | */ 638 | db->write_count = 0; 639 | rc = sqlite3_exec(db->db, "COMMIT TRANSACTION; BEGIN IMMEDIATE TRANSACTION;", 640 | NULL, NULL, &err); 641 | if (rc != 0) { 642 | fprintf(stderr, "failed to cycle sqlite transaction rc=%i: %s\n", rc, 643 | err ?: "unknown error"); 644 | /* 645 | * If we failed to cycle the transaction, it's really 646 | * not clear how the caller can recover. 647 | */ 648 | abort(); 649 | } 650 | 651 | sqlite3_free(err); 652 | return; 653 | } 654 | 655 | void 656 | proto_result_list_reset(struct proto_result_list *list) 657 | { 658 | 659 | for (size_t i = 0; i < list->count; i++) { 660 | ProtobufCMessage *proto; 661 | 662 | proto = list->rows[i].proto; 663 | protobuf_c_message_free_unpacked(proto, /*allocator=*/NULL); 664 | free(list->rows[i].bytes); 665 | } 666 | 667 | free(list->rows); 668 | 669 | *list = PROTO_RESULT_LIST_INITIALIZER; 670 | return; 671 | } 672 | 673 | static bool 674 | result_list_grow(struct proto_result_list *dst) 675 | { 676 | 677 | while (dst->count >= dst->capacity) { 678 | struct proto_result_row *grown; 679 | size_t current = dst->capacity; 680 | size_t goal = 2 * current; 681 | 682 | if (goal < 8) 683 | goal = 8; 684 | 685 | if (goal < current || goal > SIZE_MAX / sizeof(grown[0])) 686 | return false; 687 | 688 | grown = realloc(dst->rows, goal * sizeof(grown[0])); 689 | if (grown == NULL) 690 | return false; 691 | 692 | dst->rows = grown; 693 | dst->capacity = goal; 694 | } 695 | 696 | return true; 697 | } 698 | 699 | static bool 700 | result_list_push(struct proto_result_list *dst, int64_t row, void *proto, 701 | void *bytes, size_t n_bytes) 702 | { 703 | 704 | if (dst->count >= dst->capacity && result_list_grow(dst) == false) 705 | return false; 706 | 707 | dst->rows[dst->count++] = (struct proto_result_row) { 708 | .id = row, 709 | .proto = proto, 710 | .bytes = bytes, 711 | .n_bytes = n_bytes, 712 | }; 713 | 714 | return true; 715 | } 716 | 717 | int 718 | proto_result_list_populate(struct proto_result_list *dst, 719 | const ProtobufCMessageDescriptor *type, sqlite3 *db, sqlite3_stmt *stmt) 720 | { 721 | 722 | while (true) { 723 | int64_t row_id; 724 | void *copy; /* of the blob bytes. */ 725 | void *parsed; 726 | const void *blob; 727 | size_t blob_size; 728 | int rc; 729 | 730 | rc = sqlite3_step(stmt); 731 | if (rc == SQLITE_DONE) 732 | return SQLITE_OK; 733 | 734 | if (rc != SQLITE_ROW) 735 | return rc; 736 | 737 | row_id = sqlite3_column_int64(stmt, 0); 738 | blob = sqlite3_column_blob(stmt, 1); 739 | switch (sqlite3_errcode(db)) { 740 | case SQLITE_OK: 741 | case SQLITE_ROW: 742 | blob_size = (size_t)sqlite3_column_bytes(stmt, 1); 743 | break; 744 | 745 | case SQLITE_RANGE: 746 | blob = NULL; 747 | blob_size = 0; 748 | break; 749 | 750 | default: 751 | return sqlite3_errcode(db); 752 | } 753 | 754 | if (blob == NULL) { 755 | parsed = NULL; 756 | copy = NULL; 757 | blob_size = 0; 758 | } else { 759 | if (type != NULL) { 760 | parsed = protobuf_c_message_unpack(type, 761 | /*allocator=*/NULL, blob_size, blob); 762 | if (parsed == NULL) 763 | return SQLITE_ROW; 764 | } else { 765 | parsed = NULL; 766 | } 767 | 768 | copy = NULL; 769 | /* Allocate one more to append a NUL terminator. */ 770 | if (blob_size < SIZE_MAX) 771 | copy = malloc(blob_size + 1); 772 | 773 | if (copy == NULL) { 774 | protobuf_c_message_free_unpacked( 775 | parsed, /*allocator=*/NULL); 776 | return SQLITE_NOMEM; 777 | } 778 | 779 | memcpy(copy, blob, blob_size); 780 | ((uint8_t *)copy)[blob_size] = '\0'; 781 | } 782 | 783 | if (result_list_push(dst, row_id, parsed, copy, blob_size) == 784 | false) { 785 | protobuf_c_message_free_unpacked(parsed, /*allocator=*/NULL); 786 | free(copy); 787 | return SQLITE_NOMEM; 788 | } 789 | } 790 | } 791 | -------------------------------------------------------------------------------- /proto_table/proto_table.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | /** 11 | * A proto table is a view over a raw sqlite table that only contains two 12 | * columns: a integer primary key id, and a proto blob of protobuf bytes. 13 | * 14 | * The proto index constructs a view where columns correspond to 15 | * values extracted from the each row's protobuf blob, with indexes 16 | * to ensure reasonable query performance. 17 | */ 18 | 19 | /** 20 | * A `proto_column` describes one view column obtained by extracting a 21 | * protobuf path from the underlying raw table. 22 | */ 23 | struct proto_column { 24 | /* The name of the view column. */ 25 | const char *name; 26 | 27 | /* 28 | * The sqlite type of the column data (e.g, TEXT or INTEGER). 29 | * 30 | * The `protobuf_extract` function is opaque to the query 31 | * planner, so we provide explicit type information by 32 | * `CAST`ing around calls to `protobuf_extract`. 33 | */ 34 | const char *type; 35 | 36 | /* The protopath to pass to `protobuf_extract`. */ 37 | const char *path; 38 | 39 | enum proto_selector_type { 40 | /* 41 | * A strong selector (the default) gets an automatic 42 | * index on its value. 43 | */ 44 | PROTO_SELECTOR_TYPE_STRONG = 0, 45 | /* 46 | * A column that's weak will not be used to select 47 | * rows by itself (i.e., it's a weak selector). It 48 | * will not be indexed automatically, but may appear 49 | * in explicit indexes. 50 | * 51 | * It's easier to remove an index than to add one, and 52 | * the only impact of letting a column be strong when 53 | * it should be weak is a constant factor slowdown on 54 | * inserts and updates. When in doubt, use the 55 | * default (strong) until there's a good reason to 56 | * avoid indexing a column. 57 | */ 58 | PROTO_SELECTOR_TYPE_WEAK = 1, 59 | } index; 60 | }; 61 | 62 | /** 63 | * A `proto_index` describes an additional index added to the raw 64 | * table. 65 | * 66 | * Each view column automatically gets an index on its expression. 67 | */ 68 | struct proto_index { 69 | /* 70 | * Unique name for the index. 71 | */ 72 | const char *name_suffix; 73 | /* 74 | * NULL terminated list of column names; any component that is 75 | * not a view column name is passed verbatim as an index 76 | * expression. 77 | */ 78 | const char *const *components; 79 | }; 80 | 81 | /** 82 | * A `proto_table` describes a view built on top of a "raw" table 83 | * (`${name}_raw`) that only has two columns: an integer primary key 84 | * `id`, and a blob of protobuf bytes, `proto`. 85 | * 86 | * From that raw table, we construct a view that extract values from 87 | * each row's proto blob; we make that's reasonably efficient with 88 | * functional indexes. 89 | */ 90 | struct proto_table { 91 | /* 92 | * The name of the view table. We will construct a prefix for 93 | * the underlying raw table and all indexes. 94 | */ 95 | const char *name; 96 | 97 | /* 98 | * Whether to log the proto table's setup SQL to stderr 99 | * whenever `proto_table_setup` constructs that SQL string 100 | * from scratch. 101 | */ 102 | bool log_sql_to_stderr; 103 | 104 | /* 105 | * The protobuf message type for all rows in this raw table. 106 | */ 107 | const char *message_name; 108 | 109 | /* 110 | * List of additional columns in the view, terminated by a 111 | * zero-filled struct. 112 | * 113 | * We automatically generate an index for each protobuf 114 | * columns (and for the integer primary key). 115 | */ 116 | const struct proto_column *columns; 117 | 118 | /* 119 | * List of additional indexes on the raw table, terminated by 120 | * a zero-filled struct. 121 | */ 122 | const struct proto_index *indexes; 123 | }; 124 | 125 | /** 126 | * It's often easier to issue a lot of small writes when working with 127 | * protobuf, which makes transactions essential for write performance. 128 | * This struct type wraps a sqlite3 db with counters for transaction 129 | * depth and *autocommit* transaction depth; as long as all 130 | * transactions on the db are for autocommit, we can flush writes 131 | * whenever the write count is too high. 132 | * 133 | * This wrapper lets caller open multiple overlapping (not necessarily 134 | * nested) transactions for the same sqlite database: the real sqlite 135 | * transaction is only affected when transitioning between 136 | * transaction_depth 1 and 0. 137 | * 138 | * The semantics are particularly useful when a function uses 139 | * transactions for correctness, and is called from another function 140 | * that opens a transaction for performance. However, it also means 141 | * that ROLLBACKs have a much wider blast radius than one might 142 | * expect. 143 | */ 144 | struct proto_db { 145 | /* 146 | * Counts the number of write operations performed 147 | * since the last transaction commit. 148 | */ 149 | uint32_t write_count; 150 | 151 | /* 152 | * Attempt to autoflush after this many write operations. 153 | * 154 | * Defaults to a reasonable batch size if 0. 155 | */ 156 | uint32_t batch_size; 157 | 158 | /* 159 | * Sqlite doesn't nest transactions, so we track the 160 | * depth on our end. 161 | */ 162 | size_t transaction_depth; 163 | 164 | /* 165 | * Counts the number of transactions that were only 166 | * created for write batch performance. When this 167 | * number is equal to the transaction depth, we can 168 | * cycle the current transaction to flush writes 169 | * whenever we want. 170 | */ 171 | size_t autocommit_depth; 172 | 173 | sqlite3 *db; 174 | }; 175 | 176 | /** 177 | * A `proto_result_row` struct represents a single result row for a 178 | * sqlite query. 179 | */ 180 | struct proto_result_row { 181 | int64_t id; /* row id */ 182 | ProtobufCMessage *proto; 183 | void *bytes; 184 | size_t n_bytes; 185 | }; 186 | 187 | struct proto_result_list { 188 | size_t count; 189 | size_t capacity; 190 | struct proto_result_row *rows; 191 | }; 192 | 193 | #define PROTO_RESULT_LIST_INITIALIZER \ 194 | (struct proto_result_list) { .count = 0 } 195 | 196 | #define PROTO_RESULT_LIST(LIST_TYPE, PROTO_TYPE, ROW_TYPE) \ 197 | struct LIST_TYPE { \ 198 | union { \ 199 | struct { \ 200 | size_t count; \ 201 | size_t capacity; \ 202 | struct ROW_TYPE { \ 203 | int64_t id; \ 204 | PROTO_TYPE *proto; \ 205 | void *bytes; \ 206 | size_t n_bytes; \ 207 | } * rows; \ 208 | }; \ 209 | struct proto_result_list impl; \ 210 | }; \ 211 | } 212 | 213 | /** 214 | * Ensures the `spec`ced table in `db` is in the expected state. 215 | * 216 | * If the `command_cache` pointee is NULL, the pointer will be 217 | * populated with a cached SQL string that corresponds to `spec`; if 218 | * its pointee is non-NULL, it must have been generated by a prior 219 | * call to `setup_proto_table` for the same `spec`. 220 | * 221 | * The sqlite_protobuf extension must be loaded before calling this 222 | * function, and the message descriptors must be present in the C++ 223 | * protobuf repository before accessing the view. 224 | * 225 | * Returns 0 (SQLITE_OK) on success, and an sqlite error code on 226 | * failure. 227 | */ 228 | int proto_table_setup( 229 | char **command_cache, sqlite3 *db, const struct proto_table *spec); 230 | 231 | /** 232 | * Find the end id for page of up to `wanted` rows in `table`, 233 | * starting at `id >= begin`. 234 | * 235 | * Returns `begin` when there is none, and a negated sqlite error code 236 | * on error. 237 | */ 238 | int64_t proto_table_paginate( 239 | sqlite3 *, const char *table, int64_t begin, size_t wanted); 240 | 241 | /** 242 | * Attempts to open a new transaction in the proto db. This wrapper 243 | * counts recursive invocation, and only opens a sqlite transaction 244 | * when the count transitions from 0 to 1. 245 | * 246 | * Rolling back is rarely a good idea using protodb. 247 | * 248 | * Returns SQLITE_OK on success, and a sqlite error code on failure. 249 | */ 250 | int proto_db_transaction_begin(struct proto_db *); 251 | 252 | /** 253 | * Closes one transaction in the proto db. This wrapper counts 254 | * recursive transactions (nested or otherwise), and only closes the 255 | * underlying sqlite transaction when the total count hits 0. 256 | * 257 | * Aborts on failure. 258 | */ 259 | void proto_db_transaction_end(struct proto_db *); 260 | 261 | /** 262 | * Attempts to open a new autocommit transaction (only for 263 | * performance) in the proto db. 264 | * 265 | * Returns SQLITE_OK on success, and a sqlite error code on failure. 266 | */ 267 | int proto_db_batch_begin(struct proto_db *); 268 | 269 | /** 270 | * Closes one autocommit transaction in the proto db. 271 | * 272 | * Aborts on failure. 273 | */ 274 | void proto_db_batch_end(struct proto_db *); 275 | 276 | /** 277 | * Updates the proto db for `count` new writes operations (rows added 278 | * or modified). 279 | * 280 | * Aborts on transaction flush failure. 281 | */ 282 | void proto_db_count_writes(struct proto_db *, size_t count); 283 | 284 | /** 285 | * Releases any resource owned by the list and reinitialises it to a 286 | * zero-filled struct. 287 | */ 288 | void proto_result_list_reset(struct proto_result_list *); 289 | 290 | /** 291 | * Pushes results from the sqlite3 statement to the proto_result_list. 292 | * 293 | * The first result column must be an integer row id, and the second 294 | * column should be a blob. If a descriptor is passed the blob is 295 | * also parsed as a protobuf message. 296 | * 297 | * Returns SQLITE_OK (0) on success and a sqlite error code on failure. 298 | * 299 | * When a descriptor is passed and we fail to parse a row's second 300 | * (blob) value as protobuf, fails with SQLITE_ROW. 301 | */ 302 | int proto_result_list_populate(struct proto_result_list *, 303 | const ProtobufCMessageDescriptor *, sqlite3 *, sqlite3_stmt *); 304 | 305 | /** 306 | * Prepares a statement for the sqlite handle `db`, and stores the 307 | * result in `stmt` on success. 308 | */ 309 | inline int 310 | proto_prepare(sqlite3 *db, sqlite3_stmt **stmt, const char *sql) 311 | { 312 | 313 | return sqlite3_prepare_v2(db, sql, strlen(sql) + 1, stmt, NULL); 314 | } 315 | 316 | /** 317 | * The PROTO_BIND{,_INDEX} make it easier to bind values to sqlite 318 | * prepared statements. 319 | */ 320 | 321 | /** 322 | * This placeholder struct describes SQL NULL values. A constant 323 | * named `sqlite_null` will have a value of type `struct 324 | * proto_bind_null` inside `PROTO_BIND`'s and `PROTO_BIND_INDEX`'s 325 | * evaluation scopes. 326 | */ 327 | struct proto_bind_null { 328 | char structs_must_have_some_field; 329 | }; 330 | 331 | /** 332 | * A `proto_bind_blob` struct describes a binary blob value 333 | * of `count` bytes starting at `bytes`. 334 | */ 335 | struct proto_bind_blob { 336 | const void *bytes; 337 | size_t count; 338 | }; 339 | 340 | /** 341 | * A `proto_bind_zeroblob` struct describes a zero-filled binary blob 342 | * of `count` bytes. 343 | */ 344 | struct proto_bind_zeroblob { 345 | size_t count; 346 | }; 347 | 348 | /** 349 | * Binds the third value argument to the `IDX`th parameter in prepared 350 | * statement `STMT`. 351 | * 352 | * Pass `sqlite_null` to bind a null, and a `proto_bind_blob` 353 | * to bind a sized byte range. C strings are bound as utf-8 strings, 354 | * integers as int64, and floating point values as double. 355 | * 356 | * When value is a pointer (`proto_bind_blob` or C strings), the pointee 357 | * must outlive the prepared statement, or at least the lifetime of the 358 | * current statement up to the next call to `sqlite3_reset` on it. 359 | */ 360 | #define PROTO_BIND_INDEX(STMT, IDX, ...) \ 361 | ({ \ 362 | sqlite3_stmt *const PBI_s = (STMT); \ 363 | const int PBI_i = (IDX); \ 364 | const struct proto_bind_null sqlite_null = { 0 }; \ 365 | __auto_type PBI_v = (__VA_ARGS__); \ 366 | \ 367 | (void)sqlite_null; \ 368 | _Generic(PBI_v, \ 369 | struct proto_bind_null: proto_bind_helper_null, \ 370 | \ 371 | char *: proto_bind_helper_cstring, \ 372 | const char *: proto_bind_helper_cstring, \ 373 | \ 374 | struct proto_bind_blob: proto_bind_helper_blob, \ 375 | struct proto_bind_zeroblob: proto_bind_helper_zeroblob,\ 376 | \ 377 | const sqlite3_value *: sqlite3_bind_value, \ 378 | sqlite3_value *: sqlite3_bind_value, \ 379 | \ 380 | int8_t: sqlite3_bind_int64, \ 381 | uint8_t: sqlite3_bind_int64, \ 382 | int16_t: sqlite3_bind_int64, \ 383 | uint16_t: sqlite3_bind_int64, \ 384 | int32_t: sqlite3_bind_int64, \ 385 | uint32_t: sqlite3_bind_int64, \ 386 | int64_t: sqlite3_bind_int64, \ 387 | uint64_t: sqlite3_bind_int64, \ 388 | \ 389 | float: sqlite3_bind_double, \ 390 | double: sqlite3_bind_double) \ 391 | (PBI_s, PBI_i, PBI_v); \ 392 | }) 393 | 394 | /** 395 | * Binds parameter `NAME` in prepared statement `STMT` to the third 396 | * argument's value. Asserts out if no such parameter can be found 397 | * in the prepared statement. 398 | * 399 | * See `SQLITE_BIND_INDEX` for the specific type dispatch on that 400 | * value. 401 | * 402 | * Values bound as strings or blobs must outlive the prepared 403 | * statement (or at least, the statement must be reset before 404 | * the values' lifetimes end). 405 | */ 406 | #define PROTO_BIND(STMT, NAME, ...) \ 407 | ({ \ 408 | sqlite3_stmt *const PB_stmt = (STMT); \ 409 | const char *const PB_name = (NAME); \ 410 | const int PB_i = sqlite3_bind_parameter_index(PB_stmt, PB_name); \ 411 | \ 412 | assert(PB_i != 0 && "parameter not found"); \ 413 | PROTO_BIND_INDEX(PB_stmt, PB_i, ##__VA_ARGS__); \ 414 | }) 415 | 416 | static inline int 417 | proto_bind_helper_null(sqlite3_stmt *stmt, int idx, struct proto_bind_null v) 418 | { 419 | 420 | (void)v; 421 | return sqlite3_bind_null(stmt, idx); 422 | } 423 | 424 | static inline int 425 | proto_bind_helper_cstring(sqlite3_stmt *stmt, int idx, const char *v) 426 | { 427 | 428 | return sqlite3_bind_text64( 429 | stmt, idx, v, strlen(v), SQLITE_STATIC, SQLITE_UTF8); 430 | } 431 | 432 | static inline int 433 | proto_bind_helper_blob(sqlite3_stmt *stmt, int idx, struct proto_bind_blob v) 434 | { 435 | 436 | return sqlite3_bind_blob64(stmt, idx, v.bytes, v.count, SQLITE_STATIC); 437 | } 438 | 439 | static inline int 440 | proto_bind_helper_zeroblob(sqlite3_stmt *stmt, int idx, struct proto_bind_zeroblob v) 441 | { 442 | 443 | return sqlite3_bind_zeroblob64(stmt, idx, v.count); 444 | } 445 | -------------------------------------------------------------------------------- /src/extension_main.cpp: -------------------------------------------------------------------------------- 1 | #include "sqlite_protobuf.h" 2 | 3 | #include "sqlite3ext.h" 4 | 5 | #include "protobuf_enum.h" 6 | #include "protobuf_extract.h" 7 | #include "protobuf_json.h" 8 | #include "protobuf_load.h" 9 | #include "protobuf_text.h" 10 | 11 | namespace sqlite_protobuf { 12 | SQLITE_EXTENSION_INIT1 13 | 14 | extern "C" 15 | int sqlite3_sqliteprotobuf_init(sqlite3 *db, 16 | char **pzErrMsg, 17 | const sqlite3_api_routines *pApi) 18 | { 19 | SQLITE_EXTENSION_INIT2(pApi); 20 | 21 | // Check that we are compatible with this version of SQLite 22 | // 3.13.0 - SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION added 23 | if (sqlite3_libversion_number() < 3013000) { 24 | *pzErrMsg = sqlite3_mprintf( 25 | "sqlite_protobuf requires SQLite 3.13.0 or later"); 26 | return SQLITE_ERROR; 27 | } 28 | 29 | // Run each register_* function and abort if any of them fails 30 | int (*register_fns[])(sqlite3 *, char **, const sqlite3_api_routines *) = { 31 | register_protobuf_enum, 32 | register_protobuf_extract, 33 | register_protobuf_json, 34 | register_protobuf_load, 35 | register_protobuf_text, 36 | }; 37 | 38 | int nfuncs = sizeof(register_fns) / sizeof(register_fns[0]); 39 | for (int i = 0; i < nfuncs; i ++) { 40 | int err = (register_fns[i])(db, pzErrMsg, pApi); 41 | if (err != SQLITE_OK) return err; 42 | } 43 | 44 | return SQLITE_OK; 45 | } 46 | 47 | } // namespace sqlite_protobuf 48 | -------------------------------------------------------------------------------- /src/protobuf_enum.cpp: -------------------------------------------------------------------------------- 1 | #include "protobuf_enum.h" 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "sqlite3ext.h" 11 | 12 | #include "utilities.h" 13 | 14 | namespace sqlite_protobuf { 15 | SQLITE_EXTENSION_INIT3 16 | 17 | namespace { 18 | using google::protobuf::DescriptorPool; 19 | using google::protobuf::EnumDescriptor; 20 | using google::protobuf::EnumValueDescriptor; 21 | 22 | 23 | // The column indexes, corresponding to the order of the columns in the CREATE 24 | // TABLE statement in xConnect 25 | enum { 26 | COLUMN_NUMBER, 27 | COLUMN_NAME, 28 | COLUMN_ENUM, 29 | }; 30 | 31 | // The indexing strategies used by MODULE_FUNC(xBestIndex) and MODULE_FUNC(xFilter) 32 | enum { 33 | LOOKUP_ALL, 34 | LOOKUP_BY_NUMBER, 35 | LOOKUP_BY_NAME, 36 | }; 37 | 38 | 39 | #define MODULE_FUNC(func) protobuf_enum ## _ ## func 40 | 41 | 42 | // enum_cursor is a subclass of sqlite3_vtab_cursor which holds metadata that 43 | // allows us to iterate over the EnumDescriptor 44 | typedef struct enum_cursor enum_cursor; 45 | struct enum_cursor { 46 | sqlite3_vtab_cursor base; 47 | const EnumDescriptor *descriptor; 48 | sqlite3_int64 index; 49 | sqlite3_int64 stopIndex; 50 | bool isInvalid; 51 | }; 52 | 53 | 54 | /// Connect to the eponymous virtual table 55 | static int MODULE_FUNC(xConnect) ( 56 | sqlite3 *db, 57 | void *pAux, 58 | int argc, const char * const *argv, 59 | sqlite3_vtab **ppVtab, 60 | char **pzErr 61 | ) { 62 | int err = sqlite3_declare_vtab(db, 63 | "CREATE TABLE tbl(" 64 | " number INTEGER PRIMARY KEY," 65 | " name TEXT," 66 | " enum TEXT HIDDEN" 67 | ")"); 68 | if (err != SQLITE_OK) return err; 69 | 70 | *ppVtab = (sqlite3_vtab *)sqlite3_malloc(sizeof(**ppVtab)); 71 | if (!*ppVtab) return SQLITE_NOMEM; 72 | bzero(*ppVtab, sizeof(**ppVtab)); 73 | 74 | return SQLITE_OK; 75 | } 76 | 77 | 78 | /// Undoes xOpen 79 | static int MODULE_FUNC(xDisconnect) (sqlite3_vtab *pVtab) 80 | { 81 | sqlite3_free(pVtab); 82 | return SQLITE_OK; 83 | } 84 | 85 | 86 | /// Constructor enum_cursor objects 87 | static int MODULE_FUNC(xOpen) (sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor) 88 | { 89 | enum_cursor *cursor = (enum_cursor *)sqlite3_malloc(sizeof(*cursor)); 90 | if (!cursor) 91 | return SQLITE_NOMEM; 92 | bzero(cursor, sizeof(*cursor)); 93 | *ppCursor = &cursor->base; 94 | return SQLITE_OK; 95 | } 96 | 97 | /// Destructor enum_cursor objects 98 | static int MODULE_FUNC(xClose) (sqlite3_vtab_cursor *cur) 99 | { 100 | sqlite3_free(cur); 101 | return SQLITE_OK; 102 | } 103 | 104 | /// Advance the index to the next enum value 105 | static int MODULE_FUNC(xNext) (sqlite3_vtab_cursor *cur) 106 | { 107 | enum_cursor *cursor = (enum_cursor *)cur; 108 | cursor->index += 1; 109 | return SQLITE_OK; 110 | } 111 | 112 | 113 | /// Returns the current index 114 | static int MODULE_FUNC(xRowid) (sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid) 115 | { 116 | enum_cursor *cursor = (enum_cursor *)cur; 117 | *pRowid = cursor->index; 118 | return SQLITE_OK; 119 | } 120 | 121 | 122 | /// Returns true if the cursor is no longer pointing at a valid enum value 123 | static int MODULE_FUNC(xEof) (sqlite3_vtab_cursor *cur) 124 | { 125 | enum_cursor *cursor = (enum_cursor *)cur; 126 | return !cursor->descriptor || cursor->isInvalid || 127 | cursor->index >= cursor->stopIndex; 128 | } 129 | 130 | 131 | /// Return the fields in a given cell of the table 132 | static int MODULE_FUNC(xColumn) ( 133 | sqlite3_vtab_cursor *cur, 134 | sqlite3_context *ctx, 135 | int i 136 | ) { 137 | enum_cursor *cursor = (enum_cursor *)cur; 138 | switch (i) { 139 | case COLUMN_NUMBER: 140 | sqlite3_result_int64(ctx, 141 | cursor->descriptor->value(cursor->index)->number()); 142 | break; 143 | case COLUMN_NAME: 144 | sqlite3_result_text(ctx, 145 | cursor->descriptor->value(cursor->index)->name().c_str(), -1, 0); 146 | break; 147 | case COLUMN_ENUM: 148 | sqlite3_result_text(ctx, 149 | cursor->descriptor->full_name().c_str(), -1, 0); 150 | break; 151 | } 152 | return SQLITE_OK; 153 | } 154 | 155 | 156 | /// 157 | static int MODULE_FUNC(xBestIndex) ( 158 | sqlite3_vtab *tab, 159 | sqlite3_index_info *pIdxInfo 160 | ) 161 | { 162 | // Loop over the constraints to find useful ones -- namely, ones that pin 163 | // a column to a specific value 164 | int numberEqConstraintIdx = -1; 165 | int nameEqConstraintIdx = -1; 166 | int enumEqConstraintIdx = -1; 167 | 168 | const auto *constraint = pIdxInfo->aConstraint; 169 | for(int i = 0; i < pIdxInfo->nConstraint; i ++, constraint ++) { 170 | if (!constraint->usable) continue; 171 | if (constraint->op == SQLITE_INDEX_CONSTRAINT_EQ) { 172 | switch(constraint->iColumn) { 173 | case COLUMN_NUMBER: 174 | numberEqConstraintIdx = i; 175 | break; 176 | case COLUMN_NAME: 177 | nameEqConstraintIdx = i; 178 | break; 179 | case COLUMN_ENUM: 180 | enumEqConstraintIdx = i; 181 | break; 182 | } 183 | } 184 | } 185 | 186 | // If we did not get a constraint on the enum type name, we cannot continue 187 | if (enumEqConstraintIdx == -1) { 188 | return SQLITE_CONSTRAINT; 189 | } 190 | 191 | // If we have a constraint on the name, we can only return zero or one 192 | // matching rows. We can't say the same for a constraint on the number, 193 | // in case aliases are allowed. This allows SQLite to be more efficient. 194 | if (nameEqConstraintIdx >= 0) { 195 | pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; 196 | } 197 | 198 | // Decide on the indexing strategy to use. Also, tell SQLite the cost of 199 | // different combinations of constraints, to help it compare multiple query 200 | // plans. 201 | if (numberEqConstraintIdx >= 0 && nameEqConstraintIdx < 0) { 202 | pIdxInfo->idxNum = LOOKUP_BY_NUMBER; 203 | pIdxInfo->estimatedCost = 1; 204 | } else if (nameEqConstraintIdx >= 0) { 205 | pIdxInfo->idxNum = LOOKUP_BY_NAME; 206 | pIdxInfo->estimatedCost = 5; 207 | } else { 208 | pIdxInfo->idxNum = LOOKUP_ALL; 209 | pIdxInfo->estimatedCost = 100; 210 | } 211 | 212 | // Copy the values of our constraints (i.e., the right-hand sides of the 213 | // equality assertions) into the arguments that will be passed to MODULE_FUNC(xFilter). 214 | // argv[0] = enum type name 215 | // argv[1] = number or name to lookup 216 | int argIdx = 1; 217 | pIdxInfo->aConstraintUsage[enumEqConstraintIdx].argvIndex = argIdx ++; 218 | switch (pIdxInfo->idxNum) { 219 | case LOOKUP_BY_NUMBER: 220 | pIdxInfo->aConstraintUsage[numberEqConstraintIdx].argvIndex = argIdx ++; 221 | break; 222 | case LOOKUP_BY_NAME: 223 | pIdxInfo->aConstraintUsage[nameEqConstraintIdx].argvIndex = argIdx ++; 224 | break; 225 | } 226 | 227 | // For some constraints, SQLite does not need to double check that our 228 | // output matches the constraints. 229 | for (int constraintIdx : { enumEqConstraintIdx, nameEqConstraintIdx }) { 230 | if (constraintIdx >= 0) { 231 | pIdxInfo->aConstraintUsage[constraintIdx].omit = 1; 232 | } 233 | } 234 | 235 | return SQLITE_OK; 236 | } 237 | 238 | 239 | /// Initialize the enum_cursor with information about the enum type, and 240 | /// position it at the start of the relevant results. 241 | static int MODULE_FUNC(xFilter) ( 242 | sqlite3_vtab_cursor *pVtabCursor, 243 | int idxNum, const char *idxStr, 244 | int argc, sqlite3_value **argv 245 | ){ 246 | enum_cursor *cursor = (enum_cursor *)pVtabCursor; 247 | 248 | // Find the descriptor for this enum type 249 | std::string enum_name = string_from_sqlite3_value(argv[0]); 250 | cursor->descriptor = 251 | DescriptorPool::generated_pool()->FindEnumTypeByName(enum_name); 252 | if (!cursor->descriptor) { 253 | // TODO: Better way to report this error message? 254 | sqlite3_log(SQLITE_WARNING, "Could not find enum type \"%s\"", 255 | enum_name.c_str()); 256 | return SQLITE_ERROR; 257 | } 258 | 259 | // Set the cursor to iterate over the whole set of enum values, but this 260 | // may be overridden by a specific strategy. 261 | cursor->index = 0; 262 | cursor->stopIndex = cursor->descriptor->value_count(); 263 | 264 | // Find the value by our constraint argument 265 | const EnumValueDescriptor *value_desc; 266 | switch (idxNum) { 267 | // If there's no particular index strategy, we don't need to do anything 268 | // further. 269 | case LOOKUP_ALL: 270 | return SQLITE_OK; 271 | case LOOKUP_BY_NUMBER: 272 | value_desc = cursor->descriptor->FindValueByNumber( 273 | sqlite3_value_int(argv[1])); 274 | break; 275 | case LOOKUP_BY_NAME: 276 | value_desc = cursor->descriptor->FindValueByName( 277 | string_from_sqlite3_value(argv[1])); 278 | break; 279 | default: 280 | value_desc = nullptr; 281 | break; 282 | } 283 | 284 | if (!value_desc) { 285 | cursor->isInvalid = 1; 286 | return SQLITE_OK; 287 | } 288 | 289 | // Set the starting index to the index of the result 290 | cursor->index = value_desc->index(); 291 | 292 | if (idxNum == LOOKUP_BY_NUMBER && 293 | cursor->descriptor->options().allow_alias()) { 294 | // In this case, we cannot set an upper bound, because we need there 295 | // may be additional values with the same number. 296 | } else { 297 | // Set the upper bound to the next entry 298 | cursor->stopIndex = cursor->index + 1; 299 | } 300 | 301 | return SQLITE_OK; 302 | } 303 | 304 | 305 | static sqlite3_module module = { 306 | 0, /* iVersion */ 307 | 0, /* xCreate */ 308 | MODULE_FUNC(xConnect), /* xConnect - required */ 309 | MODULE_FUNC(xBestIndex), /* xBestIndex - required */ 310 | MODULE_FUNC(xDisconnect), /* xDisconnect - required */ 311 | 0, /* xDestroy */ 312 | MODULE_FUNC(xOpen), /* xOpen - open a cursor - required */ 313 | MODULE_FUNC(xClose), /* xClose - close a cursor - required */ 314 | MODULE_FUNC(xFilter), /* xFilter - configure scan constraints - required */ 315 | MODULE_FUNC(xNext), /* xNext - advance a cursor - required */ 316 | MODULE_FUNC(xEof), /* xEof - check for end of scan - required */ 317 | MODULE_FUNC(xColumn), /* xColumn - read data - required */ 318 | MODULE_FUNC(xRowid), /* xRowid - read data - required */ 319 | 0, /* xUpdate */ 320 | 0, /* xBegin */ 321 | 0, /* xSync */ 322 | 0, /* xCommit */ 323 | 0, /* xRollback */ 324 | 0, /* xFindMethod */ 325 | 0, /* xRename */ 326 | }; 327 | 328 | } // namespace 329 | 330 | int 331 | register_protobuf_enum(sqlite3 *db, char **pzErrMsg, 332 | const sqlite3_api_routines *pApi) 333 | { 334 | return sqlite3_create_module(db, "protobuf_enum", &module, 0); 335 | } 336 | 337 | } // namespace sqlite_protobuf 338 | -------------------------------------------------------------------------------- /src/protobuf_enum.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | struct sqlite3; 4 | struct sqlite3_api_routines; 5 | 6 | namespace sqlite_protobuf { 7 | 8 | int register_protobuf_enum(sqlite3 *db, char **pzErrMsg, 9 | const sqlite3_api_routines *pApi); 10 | 11 | } // namespace sqlite_protobuf 12 | -------------------------------------------------------------------------------- /src/protobuf_extract.cpp: -------------------------------------------------------------------------------- 1 | #include "protobuf_extract.h" 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | #include "sqlite3ext.h" 10 | 11 | #include "utilities.h" 12 | 13 | namespace sqlite_protobuf { 14 | SQLITE_EXTENSION_INIT3 15 | 16 | namespace { 17 | 18 | using google::protobuf::Descriptor; 19 | using google::protobuf::EnumDescriptor; 20 | using google::protobuf::EnumValueDescriptor; 21 | using google::protobuf::FieldDescriptor; 22 | using google::protobuf::Message; 23 | using google::protobuf::Reflection; 24 | 25 | 26 | /// For enum fields, handle the special suffix paths .name and .number 27 | static bool handle_special_enum_path(sqlite3_context *context, 28 | const EnumDescriptor *enum_descriptor, 29 | int value, 30 | const std::string& path, 31 | std::string::const_iterator& it) 32 | { 33 | // Get the remainder of the path 34 | std::string rest = path.substr(std::distance(path.begin(), it)); 35 | 36 | if (rest == "" || rest == ".number") 37 | { 38 | sqlite3_result_int64(context, value); 39 | return true; 40 | } 41 | else if (rest == ".name") 42 | { 43 | const EnumValueDescriptor *value_descriptor = 44 | enum_descriptor->FindValueByNumber(value); 45 | if (!value_descriptor) 46 | { 47 | sqlite3_result_error(context, "Enum value not found", -1); 48 | return false; 49 | } 50 | 51 | sqlite3_result_text(context, 52 | value_descriptor->name().c_str(), 53 | value_descriptor->name().length(), 54 | SQLITE_TRANSIENT); 55 | return true; 56 | } 57 | 58 | // This error message should match what happens for non-enums also 59 | sqlite3_result_error(context, "Path traverses non-message elements", -1); 60 | return false; 61 | } 62 | 63 | 64 | /// Return the element (or elements) 65 | /// 66 | /// SELECT protobuf_extract(data, "Person", "$.phones[0].number", default?); 67 | /// 68 | /// @returns a Protobuf-encoded BLOB or the appropriate SQL datatype 69 | /// 70 | /// If `default` is provided, it is returned instead of the protobuf field's 71 | /// default value. 72 | static void protobuf_extract(sqlite3_context *context, 73 | int argc, 74 | sqlite3_value **argv) 75 | { 76 | if (argc < 3 || argc > 4) { 77 | sqlite3_result_error( 78 | context, 79 | "wrong number of arguments to function protobuf_extract (expected 3 or 4)", 80 | SQLITE_CONSTRAINT_FUNCTION); 81 | return; 82 | } 83 | 84 | sqlite3_value *message_data = argv[0]; 85 | sqlite3_value *message_name = argv[1]; 86 | const std::string path = string_from_sqlite3_value(argv[2]); 87 | sqlite3_value *const default_value = (argc >= 4) ? argv[3] : nullptr; 88 | 89 | // Check that the path begins with $, representing the root of the tree 90 | if (path.length() == 0 || path[0] != '$') { 91 | sqlite3_result_error(context, "Invalid path", -1); 92 | return; 93 | } 94 | 95 | // Deserialize the message 96 | auto root_message = parse_message(context, message_data, message_name); 97 | if (!root_message) { 98 | return; 99 | } 100 | 101 | // Special case: just return the root object 102 | if (path == "$") { 103 | const void *data = sqlite3_value_blob(message_data); 104 | int len = sqlite3_value_bytes(message_data); 105 | sqlite3_result_blob(context, data, len, SQLITE_TRANSIENT); 106 | return; 107 | } 108 | 109 | // Get the Descrptor interface for the message type 110 | const Descriptor* descriptor = root_message->GetDescriptor(); 111 | 112 | // Get the Reflection interface for the message 113 | const Reflection *reflection = root_message->GetReflection(); 114 | 115 | // As we traverse the tree, this is the "current" message we are looking at. 116 | // We only want the overall message to be managed by std::unique_ptr, 117 | // and this variable will always point into the overall structure. 118 | const Message *message = root_message; 119 | 120 | // Parse the rest 121 | static const auto& path_element_regex = 122 | *new std::regex("^\\.([^\\.\\[]+)(?:\\[(-?[0-9]+)\\])?"); 123 | std::string::const_iterator it = ++ path.cbegin(); // skip $ 124 | while (it != path.end()) { 125 | std::smatch m; 126 | if (!std::regex_search(it, path.cend(), m, path_element_regex)) { 127 | sqlite3_result_error(context, "Invalid path", -1); 128 | return; 129 | } 130 | 131 | // Advance the iterator to the start of the next path component 132 | it += m.length(); 133 | 134 | const std::string field_name = m.str(1); 135 | const std::string field_index_str = m.str(2); 136 | 137 | // Get the descriptor for this field by its name 138 | const FieldDescriptor *const field = 139 | descriptor->FindFieldByName(field_name); 140 | if (!field) { 141 | sqlite3_result_error(context, "Invalid field name", -1); 142 | return; 143 | } 144 | 145 | // If the field is optional, and it is not provided, return the default 146 | if (field->is_optional() && !reflection->HasField(*message, field)) { 147 | // Is there anything left in the path? 148 | if (it != path.end()) { 149 | switch (field->type()) 150 | { 151 | case FieldDescriptor::Type::TYPE_ENUM: 152 | case FieldDescriptor::Type::TYPE_MESSAGE: 153 | // Both of these are handled in the main switch below 154 | break; 155 | default: 156 | sqlite3_result_error(context, "Invalid path", -1); 157 | return; 158 | } 159 | } 160 | 161 | if (default_value != nullptr) { 162 | sqlite3_result_value(context, default_value); 163 | return; 164 | } 165 | 166 | switch(field->cpp_type()) { 167 | case FieldDescriptor::CppType::CPPTYPE_INT32: 168 | sqlite3_result_int64(context, field->default_value_int32()); 169 | return; 170 | case FieldDescriptor::CppType::CPPTYPE_INT64: 171 | sqlite3_result_int64(context, field->default_value_int64()); 172 | return; 173 | case FieldDescriptor::CppType::CPPTYPE_UINT32: 174 | sqlite3_result_int64(context, field->default_value_uint32()); 175 | return; 176 | case FieldDescriptor::CppType::CPPTYPE_UINT64: 177 | sqlite3_log(SQLITE_WARNING, 178 | "Protobuf field \"%s\" is unsigned, but SQLite does not " 179 | "support unsigned types", field->full_name().c_str()); 180 | sqlite3_result_int64(context, field->default_value_uint64()); 181 | return; 182 | case FieldDescriptor::CppType::CPPTYPE_DOUBLE: 183 | sqlite3_result_double(context, field->default_value_double()); 184 | return; 185 | case FieldDescriptor::CppType::CPPTYPE_FLOAT: 186 | sqlite3_result_double(context, field->default_value_float()); 187 | return; 188 | case FieldDescriptor::CppType::CPPTYPE_BOOL: 189 | sqlite3_result_int64(context, 190 | field->default_value_bool() ? 0 : 1); 191 | return; 192 | case FieldDescriptor::CppType::CPPTYPE_ENUM: 193 | handle_special_enum_path(context, 194 | field->default_value_enum()->type(), 195 | field->default_value_enum()->number(), 196 | path, it); 197 | return; 198 | case FieldDescriptor::CppType::CPPTYPE_STRING: 199 | switch(field->type()) { 200 | default: 201 | // fall through, but log 202 | sqlite3_log(SQLITE_WARNING, 203 | "Protobuf field \"%s\" is an unexpected string type", 204 | field->full_name().c_str()); 205 | case FieldDescriptor::Type::TYPE_STRING: 206 | sqlite3_result_text(context, 207 | field->default_value_string().c_str(), 208 | field->default_value_string().length(), 209 | SQLITE_TRANSIENT); 210 | break; 211 | case FieldDescriptor::Type::TYPE_BYTES: 212 | sqlite3_result_blob(context, 213 | field->default_value_string().c_str(), 214 | field->default_value_string().length(), 215 | SQLITE_TRANSIENT); 216 | break; 217 | } 218 | return; 219 | case FieldDescriptor::CppType::CPPTYPE_MESSAGE: 220 | sqlite3_result_null(context); 221 | return; 222 | } 223 | } 224 | 225 | int field_index = 0; 226 | const bool is_repeated = field->is_repeated(); 227 | // If the field is repeated, validate the index into it 228 | if (is_repeated) { 229 | if (field_index_str.empty()) { 230 | sqlite3_result_error(context, 231 | "Expected index into repeated field", -1); 232 | return; 233 | } 234 | 235 | // Wrap around for negative indexing 236 | int field_size = reflection->FieldSize(*message, field); 237 | field_index = std::stoi(field_index_str); 238 | 239 | if (field_index < 0) { 240 | field_index = field_size + field_index; 241 | } 242 | 243 | // Check that it's within range 244 | if (field_index < 0 || field_index >= field_size) { 245 | // If we error here, that means the query will stop 246 | sqlite3_result_null(context); 247 | return; 248 | } 249 | } 250 | 251 | // If the field is a submessage, descend into it 252 | if (field->cpp_type() == FieldDescriptor::CppType::CPPTYPE_MESSAGE) { 253 | // Descend into this submessage 254 | message = is_repeated 255 | ? &reflection->GetRepeatedMessage(*message, field, field_index) 256 | : &reflection->GetMessage(*message, field); 257 | descriptor = message->GetDescriptor(); 258 | reflection = message->GetReflection(); 259 | continue; 260 | } 261 | 262 | // Any other type should be the end of the path 263 | if (it != path.cend() 264 | && field->type() != FieldDescriptor::Type::TYPE_ENUM) 265 | { 266 | sqlite3_result_error(context, "Path traverses non-message elements", 267 | -1); 268 | return; 269 | } 270 | 271 | // Translate the field type into a SQLite type and return it 272 | switch(field->cpp_type()) { 273 | case FieldDescriptor::CppType::CPPTYPE_INT32: 274 | { 275 | int32_t value = is_repeated 276 | ? reflection->GetRepeatedInt32(*message, field, field_index) 277 | : reflection->GetInt32(*message, field); 278 | sqlite3_result_int64(context, value); 279 | return; 280 | } 281 | case FieldDescriptor::CppType::CPPTYPE_INT64: 282 | { 283 | int64_t value = is_repeated 284 | ? reflection->GetRepeatedInt64(*message, field, field_index) 285 | : reflection->GetInt64(*message, field); 286 | sqlite3_result_int64(context, value); 287 | return; 288 | } 289 | case FieldDescriptor::CppType::CPPTYPE_UINT32: 290 | { 291 | uint32_t value = is_repeated 292 | ? reflection->GetRepeatedUInt32(*message, field, field_index) 293 | : reflection->GetUInt32(*message, field); 294 | sqlite3_result_int64(context, value); 295 | return; 296 | } 297 | case FieldDescriptor::CppType::CPPTYPE_UINT64: 298 | { 299 | sqlite3_log(SQLITE_WARNING, 300 | "Protobuf field \"%s\" is unsigned, but SQLite does not " 301 | "support unsigned types"); 302 | uint64_t value = is_repeated 303 | ? reflection->GetRepeatedUInt64(*message, field, field_index) 304 | : reflection->GetUInt64(*message, field); 305 | sqlite3_result_int64(context, value); 306 | return; 307 | } 308 | case FieldDescriptor::CppType::CPPTYPE_DOUBLE: 309 | { 310 | double value = is_repeated 311 | ? reflection->GetRepeatedDouble(*message, field, field_index) 312 | : reflection->GetDouble(*message, field); 313 | sqlite3_result_double(context, value); 314 | return; 315 | } 316 | case FieldDescriptor::CppType::CPPTYPE_FLOAT: 317 | { 318 | float value = is_repeated 319 | ? reflection->GetRepeatedFloat(*message, field, field_index) 320 | : reflection->GetFloat(*message, field); 321 | sqlite3_result_double(context, value); 322 | return; 323 | } 324 | case FieldDescriptor::CppType::CPPTYPE_BOOL: 325 | { 326 | bool value = is_repeated 327 | ? reflection->GetRepeatedBool(*message, field, field_index) 328 | : reflection->GetBool(*message, field); 329 | sqlite3_result_int64(context, value ? 0 : 1); 330 | return; 331 | } 332 | case FieldDescriptor::CppType::CPPTYPE_ENUM: 333 | { 334 | int value = is_repeated 335 | ? reflection->GetRepeatedEnumValue(*message, field, field_index) 336 | : reflection->GetEnumValue(*message, field); 337 | handle_special_enum_path(context, field->enum_type(), value, 338 | path, it); 339 | return; 340 | } 341 | case FieldDescriptor::CppType::CPPTYPE_STRING: 342 | { 343 | std::string value = is_repeated 344 | ? reflection->GetRepeatedString(*message, field, field_index) 345 | : reflection->GetString(*message, field); 346 | switch(field->type()) { 347 | default: 348 | // fall through, but log 349 | sqlite3_log(SQLITE_WARNING, 350 | "Protobuf field \"%s\" is an unexpected string type", 351 | field->full_name().c_str()); 352 | case FieldDescriptor::Type::TYPE_STRING: 353 | sqlite3_result_text(context, value.c_str(), value.length(), 354 | SQLITE_TRANSIENT); 355 | break; 356 | case FieldDescriptor::Type::TYPE_BYTES: 357 | sqlite3_result_blob(context, value.c_str(), value.length(), 358 | SQLITE_TRANSIENT); 359 | break; 360 | } 361 | return; 362 | } 363 | case FieldDescriptor::CppType::CPPTYPE_MESSAGE: 364 | // Covered separately above, silence the warning 365 | break; 366 | } 367 | } 368 | 369 | // We made it to the end of the path. This means the user selected for a 370 | // message, which we should return the Protobuf-encoded message we landed on 371 | std::string serialized; 372 | if (!message->SerializeToString(&serialized)) { 373 | sqlite3_result_error(context, "Could not serialize message", -1); 374 | return; 375 | } 376 | sqlite3_result_blob(context, serialized.c_str(), serialized.length(), 377 | SQLITE_TRANSIENT); 378 | } 379 | } // namespace 380 | 381 | int 382 | register_protobuf_extract(sqlite3 *db, char **pzErrMsg, 383 | const sqlite3_api_routines *pApi) 384 | { 385 | return sqlite3_create_function(db, "protobuf_extract", -1, 386 | SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, protobuf_extract, 0, 0); 387 | } 388 | 389 | } // namespace sqlite_protobuf 390 | -------------------------------------------------------------------------------- /src/protobuf_extract.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | struct sqlite3; 4 | struct sqlite3_api_routines; 5 | 6 | namespace sqlite_protobuf { 7 | 8 | int register_protobuf_extract(sqlite3 *db, char **pzErrMsg, 9 | const sqlite3_api_routines *pApi); 10 | 11 | } // namespace sqlite_protobuf 12 | -------------------------------------------------------------------------------- /src/protobuf_json.cpp: -------------------------------------------------------------------------------- 1 | #include "protobuf_json.h" 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "sqlite3ext.h" 11 | 12 | #include "utilities.h" 13 | 14 | namespace sqlite_protobuf { 15 | SQLITE_EXTENSION_INIT3 16 | 17 | namespace { 18 | 19 | using google::protobuf::Message; 20 | using google::protobuf::util::JsonParseOptions; 21 | using google::protobuf::util::JsonPrintOptions; 22 | using google::protobuf::util::JsonStringToMessage; 23 | using google::protobuf::util::MessageToJsonString; 24 | 25 | 26 | /// Converts a binary blob of protobuf bytes to a JSON representation of the message. 27 | /// 28 | /// SELECT protobuf_to_json(data, "Person"); 29 | /// 30 | /// @returns a JSON string. 31 | void 32 | protobuf_to_json(sqlite3_context *context, int argc, sqlite3_value **argv) 33 | { 34 | sqlite3_value *message_data = argv[0]; 35 | sqlite3_value *message_name = argv[1]; 36 | 37 | auto message = parse_message(context, message_data, message_name); 38 | if (!message) { 39 | return; 40 | } 41 | 42 | JsonPrintOptions options; 43 | // The JSON format is unfortunately tied to proto3 semantics, 44 | // where there is no difference between unpopulated primitive 45 | // fields and primitive fields set to their default value. We may 46 | // parse this JSON in languages like C or Javascript that make it 47 | // easy to miss a null check, so we prefer to always populate 48 | // fields we know about. 49 | options.always_print_primitive_fields = true; 50 | 51 | std::string json; 52 | if (!MessageToJsonString(*message, &json, options).ok()) { 53 | sqlite3_result_error(context, "Could not convert message to JSON", -1); 54 | return; 55 | } 56 | 57 | sqlite3_result_text(context, json.c_str(), json.length(), SQLITE_TRANSIENT); 58 | return; 59 | } 60 | 61 | /// Converts a JSON string to a binary blob of protobuf bytes. 62 | /// 63 | /// SELECT protobuf_of_json(json, "Person"); 64 | /// 65 | /// @returns a protobuf blob. 66 | void 67 | protobuf_of_json(sqlite3_context *context, int argc, sqlite3_value **argv) 68 | { 69 | const std::string json_data = string_from_sqlite3_value(argv[0]); 70 | sqlite3_value *message_name = argv[1]; 71 | 72 | const Message *prototype = get_prototype(context, message_name); 73 | if (!prototype) { 74 | return; 75 | } 76 | 77 | std::unique_ptr message(prototype->New()); 78 | 79 | JsonParseOptions options; 80 | options.ignore_unknown_fields = true; 81 | 82 | if (!JsonStringToMessage(json_data, message.get(), options).ok()) { 83 | sqlite3_result_error(context, "Could not parse JSON message", -1); 84 | return; 85 | } 86 | 87 | std::string proto; 88 | if (!message->SerializeToString(&proto)) { 89 | sqlite3_result_error(context, "Could not serialize message", -1); 90 | return; 91 | } 92 | 93 | sqlite3_result_blob(context, proto.c_str(), proto.length(), SQLITE_TRANSIENT); 94 | return; 95 | } 96 | 97 | } // namespace 98 | 99 | int 100 | register_protobuf_json(sqlite3 *db, char **pzErrMsg, 101 | const sqlite3_api_routines *pApi) 102 | { 103 | int rc; 104 | 105 | rc = sqlite3_create_function(db, "protobuf_to_json", 2, 106 | SQLITE_UTF8 | SQLITE_DETERMINISTIC, 107 | nullptr, protobuf_to_json, nullptr, nullptr); 108 | if (rc != SQLITE_OK) 109 | return rc; 110 | 111 | return sqlite3_create_function(db, "protobuf_of_json", 2, 112 | SQLITE_UTF8 | SQLITE_DETERMINISTIC, 113 | nullptr, protobuf_of_json, nullptr, nullptr); 114 | } 115 | 116 | } // namespace sqlite_protobuf 117 | -------------------------------------------------------------------------------- /src/protobuf_json.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | struct sqlite3; 4 | struct sqlite3_api_routines; 5 | 6 | namespace sqlite_protobuf { 7 | 8 | int register_protobuf_json(sqlite3 *db, char **pzErrMsg, 9 | const sqlite3_api_routines *pApi); 10 | 11 | } // namespace sqlite_protobuf 12 | -------------------------------------------------------------------------------- /src/protobuf_load.cpp: -------------------------------------------------------------------------------- 1 | #include "protobuf_load.h" 2 | 3 | #include 4 | 5 | #include 6 | 7 | #include "sqlite3ext.h" 8 | 9 | #include "utilities.h" 10 | 11 | namespace sqlite_protobuf { 12 | SQLITE_EXTENSION_INIT3 13 | 14 | namespace { 15 | /// Loads a shared library that presumably contains message descriptors. Returns 16 | /// NULL on success or throws an error on failure. 17 | /// 18 | /// SELECT protobuf_load("example/libaddressbook.dylib"); 19 | /// 20 | static void protobuf_load(sqlite3_context *context, 21 | int argc, 22 | sqlite3_value **argv) 23 | { 24 | invalidate_all_caches(); 25 | 26 | // Confirm that we have permission to load extensions 27 | int enabled, err; 28 | err = sqlite3_db_config(sqlite3_context_db_handle(context), 29 | SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, -1, &enabled); 30 | if (err != SQLITE_OK) { 31 | auto error_msg = std::string("Failed to get load_extension setting: ") 32 | + sqlite3_errmsg(sqlite3_context_db_handle(context)); 33 | sqlite3_result_error(context, error_msg.c_str(), -1); 34 | return; 35 | } else if (!enabled) { 36 | sqlite3_result_error(context, "Extension loading is disabled", -1); 37 | return; 38 | } 39 | 40 | // Load the library 41 | const std::string path = string_from_sqlite3_value(argv[0]); 42 | void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_GLOBAL); 43 | if (!handle) { 44 | auto error_msg = std::string("Could not load library: ") + dlerror(); 45 | sqlite3_result_error(context, error_msg.c_str(), -1); 46 | return; 47 | } 48 | 49 | sqlite3_result_null(context); 50 | } 51 | 52 | } // namespace 53 | 54 | int 55 | register_protobuf_load(sqlite3 *db, char **pzErrMsg, 56 | const sqlite3_api_routines *pApi) 57 | { 58 | return sqlite3_create_function(db, "protobuf_load", 1, SQLITE_UTF8, 0, 59 | protobuf_load, 0, 0); 60 | } 61 | 62 | } // namespace sqlite_protobuf 63 | -------------------------------------------------------------------------------- /src/protobuf_load.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | struct sqlite3; 4 | struct sqlite3_api_routines; 5 | 6 | namespace sqlite_protobuf { 7 | 8 | int register_protobuf_load(sqlite3 *db, char **pzErrMsg, 9 | const sqlite3_api_routines *pApi); 10 | 11 | } // namespace sqlite_protobuf 12 | -------------------------------------------------------------------------------- /src/protobuf_text.cpp: -------------------------------------------------------------------------------- 1 | #include "protobuf_text.h" 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include "sqlite3ext.h" 11 | 12 | #include "utilities.h" 13 | 14 | namespace sqlite_protobuf { 15 | SQLITE_EXTENSION_INIT3 16 | 17 | namespace { 18 | 19 | using google::protobuf::Message; 20 | using google::protobuf::TextFormat; 21 | 22 | 23 | /// Converts a binary blob of protobuf bytes to text proto. 24 | /// 25 | /// SELECT protobuf_to_text(data, "Person"); 26 | /// 27 | /// @returns a text proto string. 28 | void 29 | protobuf_to_text(sqlite3_context *context, int argc, sqlite3_value **argv) 30 | { 31 | sqlite3_value *message_data = argv[0]; 32 | sqlite3_value *message_name = argv[1]; 33 | 34 | auto message = parse_message(context, message_data, message_name); 35 | if (!message) { 36 | return; 37 | } 38 | 39 | std::string text; 40 | if (!TextFormat::PrintToString(*message, &text)) { 41 | sqlite3_result_error(context, "Could not convert message to textproto", -1); 42 | return; 43 | } 44 | 45 | sqlite3_result_text(context, text.c_str(), text.length(), SQLITE_TRANSIENT); 46 | return; 47 | } 48 | 49 | /// Converts a text proto string to a binary blob of protobuf bytes. 50 | /// 51 | /// SELECT protobuf_of_text(text_proto, "Person"); 52 | /// 53 | /// @returns a protobuf blob. 54 | void 55 | protobuf_of_text(sqlite3_context *context, int argc, sqlite3_value **argv) 56 | { 57 | const std::string text_data = string_from_sqlite3_value(argv[0]); 58 | sqlite3_value *message_name = argv[1]; 59 | 60 | const Message *prototype = get_prototype(context, message_name); 61 | if (!prototype) { 62 | return; 63 | } 64 | 65 | std::unique_ptr message(prototype->New()); 66 | if (!TextFormat::ParseFromString(text_data, message.get())) { 67 | sqlite3_result_error(context, "Could not parse text proto", -1); 68 | return; 69 | } 70 | 71 | std::string proto; 72 | if (!message->SerializeToString(&proto)) { 73 | sqlite3_result_error(context, "Could not serialize message", -1); 74 | return; 75 | } 76 | 77 | sqlite3_result_blob(context, proto.c_str(), proto.length(), SQLITE_TRANSIENT); 78 | return; 79 | } 80 | 81 | } // namespace 82 | 83 | int 84 | register_protobuf_text(sqlite3 *db, char **pzErrMsg, 85 | const sqlite3_api_routines *pApi) 86 | { 87 | int rc; 88 | 89 | rc = sqlite3_create_function(db, "protobuf_to_text", 2, 90 | SQLITE_UTF8 | SQLITE_DETERMINISTIC, 91 | nullptr, protobuf_to_text, nullptr, nullptr); 92 | if (rc != SQLITE_OK) 93 | return rc; 94 | 95 | return sqlite3_create_function(db, "protobuf_of_text", 2, 96 | SQLITE_UTF8 | SQLITE_DETERMINISTIC, 97 | nullptr, protobuf_of_text, nullptr, nullptr); 98 | } 99 | 100 | } // namespace sqlite_protobuf 101 | -------------------------------------------------------------------------------- /src/protobuf_text.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | struct sqlite3; 4 | struct sqlite3_api_routines; 5 | 6 | namespace sqlite_protobuf { 7 | 8 | int register_protobuf_text(sqlite3 *db, char **pzErrMsg, 9 | const sqlite3_api_routines *pApi); 10 | 11 | } // namespace sqlite_protobuf 12 | -------------------------------------------------------------------------------- /src/sqlite3ext.h: -------------------------------------------------------------------------------- 1 | /* 2 | ** 2006 June 7 3 | ** 4 | ** The author disclaims copyright to this source code. In place of 5 | ** a legal notice, here is a blessing: 6 | ** 7 | ** May you do good and not evil. 8 | ** May you find forgiveness for yourself and forgive others. 9 | ** May you share freely, never taking more than you give. 10 | ** 11 | ************************************************************************* 12 | ** This header file defines the SQLite interface for use by 13 | ** shared libraries that want to be imported as extensions into 14 | ** an SQLite instance. Shared libraries that intend to be loaded 15 | ** as extensions by SQLite should #include this file instead of 16 | ** sqlite3.h. 17 | */ 18 | #ifndef _SQLITE3EXT_H_ 19 | #define _SQLITE3EXT_H_ 20 | #include "sqlite3.h" 21 | 22 | typedef struct sqlite3_api_routines sqlite3_api_routines; 23 | 24 | /* 25 | ** The following structure holds pointers to all of the SQLite API 26 | ** routines. 27 | ** 28 | ** WARNING: In order to maintain backwards compatibility, add new 29 | ** interfaces to the end of this structure only. If you insert new 30 | ** interfaces in the middle of this structure, then older different 31 | ** versions of SQLite will not be able to load each other's shared 32 | ** libraries! 33 | */ 34 | struct sqlite3_api_routines { 35 | void * (*aggregate_context)(sqlite3_context*,int nBytes); 36 | int (*aggregate_count)(sqlite3_context*); 37 | int (*bind_blob)(sqlite3_stmt*,int,const void*,int n,void(*)(void*)); 38 | int (*bind_double)(sqlite3_stmt*,int,double); 39 | int (*bind_int)(sqlite3_stmt*,int,int); 40 | int (*bind_int64)(sqlite3_stmt*,int,sqlite_int64); 41 | int (*bind_null)(sqlite3_stmt*,int); 42 | int (*bind_parameter_count)(sqlite3_stmt*); 43 | int (*bind_parameter_index)(sqlite3_stmt*,const char*zName); 44 | const char * (*bind_parameter_name)(sqlite3_stmt*,int); 45 | int (*bind_text)(sqlite3_stmt*,int,const char*,int n,void(*)(void*)); 46 | int (*bind_text16)(sqlite3_stmt*,int,const void*,int,void(*)(void*)); 47 | int (*bind_value)(sqlite3_stmt*,int,const sqlite3_value*); 48 | int (*busy_handler)(sqlite3*,int(*)(void*,int),void*); 49 | int (*busy_timeout)(sqlite3*,int ms); 50 | int (*changes)(sqlite3*); 51 | int (*close)(sqlite3*); 52 | int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*, 53 | int eTextRep,const char*)); 54 | int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*, 55 | int eTextRep,const void*)); 56 | const void * (*column_blob)(sqlite3_stmt*,int iCol); 57 | int (*column_bytes)(sqlite3_stmt*,int iCol); 58 | int (*column_bytes16)(sqlite3_stmt*,int iCol); 59 | int (*column_count)(sqlite3_stmt*pStmt); 60 | const char * (*column_database_name)(sqlite3_stmt*,int); 61 | const void * (*column_database_name16)(sqlite3_stmt*,int); 62 | const char * (*column_decltype)(sqlite3_stmt*,int i); 63 | const void * (*column_decltype16)(sqlite3_stmt*,int); 64 | double (*column_double)(sqlite3_stmt*,int iCol); 65 | int (*column_int)(sqlite3_stmt*,int iCol); 66 | sqlite_int64 (*column_int64)(sqlite3_stmt*,int iCol); 67 | const char * (*column_name)(sqlite3_stmt*,int); 68 | const void * (*column_name16)(sqlite3_stmt*,int); 69 | const char * (*column_origin_name)(sqlite3_stmt*,int); 70 | const void * (*column_origin_name16)(sqlite3_stmt*,int); 71 | const char * (*column_table_name)(sqlite3_stmt*,int); 72 | const void * (*column_table_name16)(sqlite3_stmt*,int); 73 | const unsigned char * (*column_text)(sqlite3_stmt*,int iCol); 74 | const void * (*column_text16)(sqlite3_stmt*,int iCol); 75 | int (*column_type)(sqlite3_stmt*,int iCol); 76 | sqlite3_value* (*column_value)(sqlite3_stmt*,int iCol); 77 | void * (*commit_hook)(sqlite3*,int(*)(void*),void*); 78 | int (*complete)(const char*sql); 79 | int (*complete16)(const void*sql); 80 | int (*create_collation)(sqlite3*,const char*,int,void*, 81 | int(*)(void*,int,const void*,int,const void*)); 82 | int (*create_collation16)(sqlite3*,const void*,int,void*, 83 | int(*)(void*,int,const void*,int,const void*)); 84 | int (*create_function)(sqlite3*,const char*,int,int,void*, 85 | void (*xFunc)(sqlite3_context*,int,sqlite3_value**), 86 | void (*xStep)(sqlite3_context*,int,sqlite3_value**), 87 | void (*xFinal)(sqlite3_context*)); 88 | int (*create_function16)(sqlite3*,const void*,int,int,void*, 89 | void (*xFunc)(sqlite3_context*,int,sqlite3_value**), 90 | void (*xStep)(sqlite3_context*,int,sqlite3_value**), 91 | void (*xFinal)(sqlite3_context*)); 92 | int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*); 93 | int (*data_count)(sqlite3_stmt*pStmt); 94 | sqlite3 * (*db_handle)(sqlite3_stmt*); 95 | int (*declare_vtab)(sqlite3*,const char*); 96 | int (*enable_shared_cache)(int); 97 | int (*errcode)(sqlite3*db); 98 | const char * (*errmsg)(sqlite3*); 99 | const void * (*errmsg16)(sqlite3*); 100 | int (*exec)(sqlite3*,const char*,sqlite3_callback,void*,char**); 101 | int (*expired)(sqlite3_stmt*); 102 | int (*finalize)(sqlite3_stmt*pStmt); 103 | void (*free)(void*); 104 | void (*free_table)(char**result); 105 | int (*get_autocommit)(sqlite3*); 106 | void * (*get_auxdata)(sqlite3_context*,int); 107 | int (*get_table)(sqlite3*,const char*,char***,int*,int*,char**); 108 | int (*global_recover)(void); 109 | void (*interruptx)(sqlite3*); 110 | sqlite_int64 (*last_insert_rowid)(sqlite3*); 111 | const char * (*libversion)(void); 112 | int (*libversion_number)(void); 113 | void *(*malloc)(int); 114 | char * (*mprintf)(const char*,...); 115 | int (*open)(const char*,sqlite3**); 116 | int (*open16)(const void*,sqlite3**); 117 | int (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); 118 | int (*prepare16)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); 119 | void * (*profile)(sqlite3*,void(*)(void*,const char*,sqlite_uint64),void*); 120 | void (*progress_handler)(sqlite3*,int,int(*)(void*),void*); 121 | void *(*realloc)(void*,int); 122 | int (*reset)(sqlite3_stmt*pStmt); 123 | void (*result_blob)(sqlite3_context*,const void*,int,void(*)(void*)); 124 | void (*result_double)(sqlite3_context*,double); 125 | void (*result_error)(sqlite3_context*,const char*,int); 126 | void (*result_error16)(sqlite3_context*,const void*,int); 127 | void (*result_int)(sqlite3_context*,int); 128 | void (*result_int64)(sqlite3_context*,sqlite_int64); 129 | void (*result_null)(sqlite3_context*); 130 | void (*result_text)(sqlite3_context*,const char*,int,void(*)(void*)); 131 | void (*result_text16)(sqlite3_context*,const void*,int,void(*)(void*)); 132 | void (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*)); 133 | void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*)); 134 | void (*result_value)(sqlite3_context*,sqlite3_value*); 135 | void * (*rollback_hook)(sqlite3*,void(*)(void*),void*); 136 | int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*, 137 | const char*,const char*),void*); 138 | void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); 139 | char * (*snprintf)(int,char*,const char*,...); 140 | int (*step)(sqlite3_stmt*); 141 | int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*, 142 | char const**,char const**,int*,int*,int*); 143 | void (*thread_cleanup)(void); 144 | int (*total_changes)(sqlite3*); 145 | void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*); 146 | int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*); 147 | void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*, 148 | sqlite_int64),void*); 149 | void * (*user_data)(sqlite3_context*); 150 | const void * (*value_blob)(sqlite3_value*); 151 | int (*value_bytes)(sqlite3_value*); 152 | int (*value_bytes16)(sqlite3_value*); 153 | double (*value_double)(sqlite3_value*); 154 | int (*value_int)(sqlite3_value*); 155 | sqlite_int64 (*value_int64)(sqlite3_value*); 156 | int (*value_numeric_type)(sqlite3_value*); 157 | const unsigned char * (*value_text)(sqlite3_value*); 158 | const void * (*value_text16)(sqlite3_value*); 159 | const void * (*value_text16be)(sqlite3_value*); 160 | const void * (*value_text16le)(sqlite3_value*); 161 | int (*value_type)(sqlite3_value*); 162 | char *(*vmprintf)(const char*,va_list); 163 | /* Added ??? */ 164 | int (*overload_function)(sqlite3*, const char *zFuncName, int nArg); 165 | /* Added by 3.3.13 */ 166 | int (*prepare_v2)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); 167 | int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); 168 | int (*clear_bindings)(sqlite3_stmt*); 169 | /* Added by 3.4.1 */ 170 | int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*, 171 | void (*xDestroy)(void *)); 172 | /* Added by 3.5.0 */ 173 | int (*bind_zeroblob)(sqlite3_stmt*,int,int); 174 | int (*blob_bytes)(sqlite3_blob*); 175 | int (*blob_close)(sqlite3_blob*); 176 | int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64, 177 | int,sqlite3_blob**); 178 | int (*blob_read)(sqlite3_blob*,void*,int,int); 179 | int (*blob_write)(sqlite3_blob*,const void*,int,int); 180 | int (*create_collation_v2)(sqlite3*,const char*,int,void*, 181 | int(*)(void*,int,const void*,int,const void*), 182 | void(*)(void*)); 183 | int (*file_control)(sqlite3*,const char*,int,void*); 184 | sqlite3_int64 (*memory_highwater)(int); 185 | sqlite3_int64 (*memory_used)(void); 186 | sqlite3_mutex *(*mutex_alloc)(int); 187 | void (*mutex_enter)(sqlite3_mutex*); 188 | void (*mutex_free)(sqlite3_mutex*); 189 | void (*mutex_leave)(sqlite3_mutex*); 190 | int (*mutex_try)(sqlite3_mutex*); 191 | int (*open_v2)(const char*,sqlite3**,int,const char*); 192 | int (*release_memory)(int); 193 | void (*result_error_nomem)(sqlite3_context*); 194 | void (*result_error_toobig)(sqlite3_context*); 195 | int (*sleep)(int); 196 | void (*soft_heap_limit)(int); 197 | sqlite3_vfs *(*vfs_find)(const char*); 198 | int (*vfs_register)(sqlite3_vfs*,int); 199 | int (*vfs_unregister)(sqlite3_vfs*); 200 | int (*xthreadsafe)(void); 201 | void (*result_zeroblob)(sqlite3_context*,int); 202 | void (*result_error_code)(sqlite3_context*,int); 203 | int (*test_control)(int, ...); 204 | void (*randomness)(int,void*); 205 | sqlite3 *(*context_db_handle)(sqlite3_context*); 206 | int (*extended_result_codes)(sqlite3*,int); 207 | int (*limit)(sqlite3*,int,int); 208 | sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*); 209 | const char *(*sql)(sqlite3_stmt*); 210 | int (*status)(int,int*,int*,int); 211 | int (*backup_finish)(sqlite3_backup*); 212 | sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*); 213 | int (*backup_pagecount)(sqlite3_backup*); 214 | int (*backup_remaining)(sqlite3_backup*); 215 | int (*backup_step)(sqlite3_backup*,int); 216 | const char *(*compileoption_get)(int); 217 | int (*compileoption_used)(const char*); 218 | int (*create_function_v2)(sqlite3*,const char*,int,int,void*, 219 | void (*xFunc)(sqlite3_context*,int,sqlite3_value**), 220 | void (*xStep)(sqlite3_context*,int,sqlite3_value**), 221 | void (*xFinal)(sqlite3_context*), 222 | void(*xDestroy)(void*)); 223 | int (*db_config)(sqlite3*,int,...); 224 | sqlite3_mutex *(*db_mutex)(sqlite3*); 225 | int (*db_status)(sqlite3*,int,int*,int*,int); 226 | int (*extended_errcode)(sqlite3*); 227 | void (*log)(int,const char*,...); 228 | sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64); 229 | const char *(*sourceid)(void); 230 | int (*stmt_status)(sqlite3_stmt*,int,int); 231 | int (*strnicmp)(const char*,const char*,int); 232 | int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*); 233 | int (*wal_autocheckpoint)(sqlite3*,int); 234 | int (*wal_checkpoint)(sqlite3*,const char*); 235 | void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); 236 | int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); 237 | int (*vtab_config)(sqlite3*,int op,...); 238 | int (*vtab_on_conflict)(sqlite3*); 239 | /* Version 3.7.16 and later */ 240 | int (*close_v2)(sqlite3*); 241 | const char *(*db_filename)(sqlite3*,const char*); 242 | int (*db_readonly)(sqlite3*,const char*); 243 | int (*db_release_memory)(sqlite3*); 244 | const char *(*errstr)(int); 245 | int (*stmt_busy)(sqlite3_stmt*); 246 | int (*stmt_readonly)(sqlite3_stmt*); 247 | int (*stricmp)(const char*,const char*); 248 | int (*uri_boolean)(const char*,const char*,int); 249 | sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64); 250 | const char *(*uri_parameter)(const char*,const char*); 251 | char *(*vsnprintf)(int,char*,const char*,va_list); 252 | int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); 253 | /* Version 3.8.7 and later */ 254 | int (*auto_extension)(void(*)(void)); 255 | int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64, 256 | void(*)(void*)); 257 | int (*bind_text64)(sqlite3_stmt*,int,const char*,sqlite3_uint64, 258 | void(*)(void*),unsigned char); 259 | int (*cancel_auto_extension)(void(*)(void)); 260 | int (*load_extension)(sqlite3*,const char*,const char*,char**); 261 | void *(*malloc64)(sqlite3_uint64); 262 | sqlite3_uint64 (*msize)(void*); 263 | void *(*realloc64)(void*,sqlite3_uint64); 264 | void (*reset_auto_extension)(void); 265 | void (*result_blob64)(sqlite3_context*,const void*,sqlite3_uint64, 266 | void(*)(void*)); 267 | void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64, 268 | void(*)(void*), unsigned char); 269 | int (*strglob)(const char*,const char*); 270 | /* Version 3.8.11 and later */ 271 | sqlite3_value *(*value_dup)(const sqlite3_value*); 272 | void (*value_free)(sqlite3_value*); 273 | int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64); 274 | int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64); 275 | /* Version 3.9.0 and later */ 276 | unsigned int (*value_subtype)(sqlite3_value*); 277 | void (*result_subtype)(sqlite3_context*,unsigned int); 278 | /* Version 3.10.0 and later */ 279 | int (*status64)(int,sqlite3_int64*,sqlite3_int64*,int); 280 | int (*strlike)(const char*,const char*,unsigned int); 281 | int (*db_cacheflush)(sqlite3*); 282 | /* Version 3.12.0 and later */ 283 | int (*system_errno)(sqlite3*); 284 | }; 285 | 286 | /* 287 | ** The following macros redefine the API routines so that they are 288 | ** redirected through the global sqlite3_api structure. 289 | ** 290 | ** This header file is also used by the loadext.c source file 291 | ** (part of the main SQLite library - not an extension) so that 292 | ** it can get access to the sqlite3_api_routines structure 293 | ** definition. But the main library does not want to redefine 294 | ** the API. So the redefinition macros are only valid if the 295 | ** SQLITE_CORE macros is undefined. 296 | */ 297 | #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) 298 | #define sqlite3_aggregate_context sqlite3_api->aggregate_context 299 | #ifndef SQLITE_OMIT_DEPRECATED 300 | #define sqlite3_aggregate_count sqlite3_api->aggregate_count 301 | #endif 302 | #define sqlite3_bind_blob sqlite3_api->bind_blob 303 | #define sqlite3_bind_double sqlite3_api->bind_double 304 | #define sqlite3_bind_int sqlite3_api->bind_int 305 | #define sqlite3_bind_int64 sqlite3_api->bind_int64 306 | #define sqlite3_bind_null sqlite3_api->bind_null 307 | #define sqlite3_bind_parameter_count sqlite3_api->bind_parameter_count 308 | #define sqlite3_bind_parameter_index sqlite3_api->bind_parameter_index 309 | #define sqlite3_bind_parameter_name sqlite3_api->bind_parameter_name 310 | #define sqlite3_bind_text sqlite3_api->bind_text 311 | #define sqlite3_bind_text16 sqlite3_api->bind_text16 312 | #define sqlite3_bind_value sqlite3_api->bind_value 313 | #define sqlite3_busy_handler sqlite3_api->busy_handler 314 | #define sqlite3_busy_timeout sqlite3_api->busy_timeout 315 | #define sqlite3_changes sqlite3_api->changes 316 | #define sqlite3_close sqlite3_api->close 317 | #define sqlite3_collation_needed sqlite3_api->collation_needed 318 | #define sqlite3_collation_needed16 sqlite3_api->collation_needed16 319 | #define sqlite3_column_blob sqlite3_api->column_blob 320 | #define sqlite3_column_bytes sqlite3_api->column_bytes 321 | #define sqlite3_column_bytes16 sqlite3_api->column_bytes16 322 | #define sqlite3_column_count sqlite3_api->column_count 323 | #define sqlite3_column_database_name sqlite3_api->column_database_name 324 | #define sqlite3_column_database_name16 sqlite3_api->column_database_name16 325 | #define sqlite3_column_decltype sqlite3_api->column_decltype 326 | #define sqlite3_column_decltype16 sqlite3_api->column_decltype16 327 | #define sqlite3_column_double sqlite3_api->column_double 328 | #define sqlite3_column_int sqlite3_api->column_int 329 | #define sqlite3_column_int64 sqlite3_api->column_int64 330 | #define sqlite3_column_name sqlite3_api->column_name 331 | #define sqlite3_column_name16 sqlite3_api->column_name16 332 | #define sqlite3_column_origin_name sqlite3_api->column_origin_name 333 | #define sqlite3_column_origin_name16 sqlite3_api->column_origin_name16 334 | #define sqlite3_column_table_name sqlite3_api->column_table_name 335 | #define sqlite3_column_table_name16 sqlite3_api->column_table_name16 336 | #define sqlite3_column_text sqlite3_api->column_text 337 | #define sqlite3_column_text16 sqlite3_api->column_text16 338 | #define sqlite3_column_type sqlite3_api->column_type 339 | #define sqlite3_column_value sqlite3_api->column_value 340 | #define sqlite3_commit_hook sqlite3_api->commit_hook 341 | #define sqlite3_complete sqlite3_api->complete 342 | #define sqlite3_complete16 sqlite3_api->complete16 343 | #define sqlite3_create_collation sqlite3_api->create_collation 344 | #define sqlite3_create_collation16 sqlite3_api->create_collation16 345 | #define sqlite3_create_function sqlite3_api->create_function 346 | #define sqlite3_create_function16 sqlite3_api->create_function16 347 | #define sqlite3_create_module sqlite3_api->create_module 348 | #define sqlite3_create_module_v2 sqlite3_api->create_module_v2 349 | #define sqlite3_data_count sqlite3_api->data_count 350 | #define sqlite3_db_handle sqlite3_api->db_handle 351 | #define sqlite3_declare_vtab sqlite3_api->declare_vtab 352 | #define sqlite3_enable_shared_cache sqlite3_api->enable_shared_cache 353 | #define sqlite3_errcode sqlite3_api->errcode 354 | #define sqlite3_errmsg sqlite3_api->errmsg 355 | #define sqlite3_errmsg16 sqlite3_api->errmsg16 356 | #define sqlite3_exec sqlite3_api->exec 357 | #ifndef SQLITE_OMIT_DEPRECATED 358 | #define sqlite3_expired sqlite3_api->expired 359 | #endif 360 | #define sqlite3_finalize sqlite3_api->finalize 361 | #define sqlite3_free sqlite3_api->free 362 | #define sqlite3_free_table sqlite3_api->free_table 363 | #define sqlite3_get_autocommit sqlite3_api->get_autocommit 364 | #define sqlite3_get_auxdata sqlite3_api->get_auxdata 365 | #define sqlite3_get_table sqlite3_api->get_table 366 | #ifndef SQLITE_OMIT_DEPRECATED 367 | #define sqlite3_global_recover sqlite3_api->global_recover 368 | #endif 369 | #define sqlite3_interrupt sqlite3_api->interruptx 370 | #define sqlite3_last_insert_rowid sqlite3_api->last_insert_rowid 371 | #define sqlite3_libversion sqlite3_api->libversion 372 | #define sqlite3_libversion_number sqlite3_api->libversion_number 373 | #define sqlite3_malloc sqlite3_api->malloc 374 | #define sqlite3_mprintf sqlite3_api->mprintf 375 | #define sqlite3_open sqlite3_api->open 376 | #define sqlite3_open16 sqlite3_api->open16 377 | #define sqlite3_prepare sqlite3_api->prepare 378 | #define sqlite3_prepare16 sqlite3_api->prepare16 379 | #define sqlite3_prepare_v2 sqlite3_api->prepare_v2 380 | #define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 381 | #define sqlite3_profile sqlite3_api->profile 382 | #define sqlite3_progress_handler sqlite3_api->progress_handler 383 | #define sqlite3_realloc sqlite3_api->realloc 384 | #define sqlite3_reset sqlite3_api->reset 385 | #define sqlite3_result_blob sqlite3_api->result_blob 386 | #define sqlite3_result_double sqlite3_api->result_double 387 | #define sqlite3_result_error sqlite3_api->result_error 388 | #define sqlite3_result_error16 sqlite3_api->result_error16 389 | #define sqlite3_result_int sqlite3_api->result_int 390 | #define sqlite3_result_int64 sqlite3_api->result_int64 391 | #define sqlite3_result_null sqlite3_api->result_null 392 | #define sqlite3_result_text sqlite3_api->result_text 393 | #define sqlite3_result_text16 sqlite3_api->result_text16 394 | #define sqlite3_result_text16be sqlite3_api->result_text16be 395 | #define sqlite3_result_text16le sqlite3_api->result_text16le 396 | #define sqlite3_result_value sqlite3_api->result_value 397 | #define sqlite3_rollback_hook sqlite3_api->rollback_hook 398 | #define sqlite3_set_authorizer sqlite3_api->set_authorizer 399 | #define sqlite3_set_auxdata sqlite3_api->set_auxdata 400 | #define sqlite3_snprintf sqlite3_api->snprintf 401 | #define sqlite3_step sqlite3_api->step 402 | #define sqlite3_table_column_metadata sqlite3_api->table_column_metadata 403 | #define sqlite3_thread_cleanup sqlite3_api->thread_cleanup 404 | #define sqlite3_total_changes sqlite3_api->total_changes 405 | #define sqlite3_trace sqlite3_api->trace 406 | #ifndef SQLITE_OMIT_DEPRECATED 407 | #define sqlite3_transfer_bindings sqlite3_api->transfer_bindings 408 | #endif 409 | #define sqlite3_update_hook sqlite3_api->update_hook 410 | #define sqlite3_user_data sqlite3_api->user_data 411 | #define sqlite3_value_blob sqlite3_api->value_blob 412 | #define sqlite3_value_bytes sqlite3_api->value_bytes 413 | #define sqlite3_value_bytes16 sqlite3_api->value_bytes16 414 | #define sqlite3_value_double sqlite3_api->value_double 415 | #define sqlite3_value_int sqlite3_api->value_int 416 | #define sqlite3_value_int64 sqlite3_api->value_int64 417 | #define sqlite3_value_numeric_type sqlite3_api->value_numeric_type 418 | #define sqlite3_value_text sqlite3_api->value_text 419 | #define sqlite3_value_text16 sqlite3_api->value_text16 420 | #define sqlite3_value_text16be sqlite3_api->value_text16be 421 | #define sqlite3_value_text16le sqlite3_api->value_text16le 422 | #define sqlite3_value_type sqlite3_api->value_type 423 | #define sqlite3_vmprintf sqlite3_api->vmprintf 424 | #define sqlite3_vsnprintf sqlite3_api->vsnprintf 425 | #define sqlite3_overload_function sqlite3_api->overload_function 426 | #define sqlite3_prepare_v2 sqlite3_api->prepare_v2 427 | #define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 428 | #define sqlite3_clear_bindings sqlite3_api->clear_bindings 429 | #define sqlite3_bind_zeroblob sqlite3_api->bind_zeroblob 430 | #define sqlite3_blob_bytes sqlite3_api->blob_bytes 431 | #define sqlite3_blob_close sqlite3_api->blob_close 432 | #define sqlite3_blob_open sqlite3_api->blob_open 433 | #define sqlite3_blob_read sqlite3_api->blob_read 434 | #define sqlite3_blob_write sqlite3_api->blob_write 435 | #define sqlite3_create_collation_v2 sqlite3_api->create_collation_v2 436 | #define sqlite3_file_control sqlite3_api->file_control 437 | #define sqlite3_memory_highwater sqlite3_api->memory_highwater 438 | #define sqlite3_memory_used sqlite3_api->memory_used 439 | #define sqlite3_mutex_alloc sqlite3_api->mutex_alloc 440 | #define sqlite3_mutex_enter sqlite3_api->mutex_enter 441 | #define sqlite3_mutex_free sqlite3_api->mutex_free 442 | #define sqlite3_mutex_leave sqlite3_api->mutex_leave 443 | #define sqlite3_mutex_try sqlite3_api->mutex_try 444 | #define sqlite3_open_v2 sqlite3_api->open_v2 445 | #define sqlite3_release_memory sqlite3_api->release_memory 446 | #define sqlite3_result_error_nomem sqlite3_api->result_error_nomem 447 | #define sqlite3_result_error_toobig sqlite3_api->result_error_toobig 448 | #define sqlite3_sleep sqlite3_api->sleep 449 | #define sqlite3_soft_heap_limit sqlite3_api->soft_heap_limit 450 | #define sqlite3_vfs_find sqlite3_api->vfs_find 451 | #define sqlite3_vfs_register sqlite3_api->vfs_register 452 | #define sqlite3_vfs_unregister sqlite3_api->vfs_unregister 453 | #define sqlite3_threadsafe sqlite3_api->xthreadsafe 454 | #define sqlite3_result_zeroblob sqlite3_api->result_zeroblob 455 | #define sqlite3_result_error_code sqlite3_api->result_error_code 456 | #define sqlite3_test_control sqlite3_api->test_control 457 | #define sqlite3_randomness sqlite3_api->randomness 458 | #define sqlite3_context_db_handle sqlite3_api->context_db_handle 459 | #define sqlite3_extended_result_codes sqlite3_api->extended_result_codes 460 | #define sqlite3_limit sqlite3_api->limit 461 | #define sqlite3_next_stmt sqlite3_api->next_stmt 462 | #define sqlite3_sql sqlite3_api->sql 463 | #define sqlite3_status sqlite3_api->status 464 | #define sqlite3_backup_finish sqlite3_api->backup_finish 465 | #define sqlite3_backup_init sqlite3_api->backup_init 466 | #define sqlite3_backup_pagecount sqlite3_api->backup_pagecount 467 | #define sqlite3_backup_remaining sqlite3_api->backup_remaining 468 | #define sqlite3_backup_step sqlite3_api->backup_step 469 | #define sqlite3_compileoption_get sqlite3_api->compileoption_get 470 | #define sqlite3_compileoption_used sqlite3_api->compileoption_used 471 | #define sqlite3_create_function_v2 sqlite3_api->create_function_v2 472 | #define sqlite3_db_config sqlite3_api->db_config 473 | #define sqlite3_db_mutex sqlite3_api->db_mutex 474 | #define sqlite3_db_status sqlite3_api->db_status 475 | #define sqlite3_extended_errcode sqlite3_api->extended_errcode 476 | #define sqlite3_log sqlite3_api->log 477 | #define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64 478 | #define sqlite3_sourceid sqlite3_api->sourceid 479 | #define sqlite3_stmt_status sqlite3_api->stmt_status 480 | #define sqlite3_strnicmp sqlite3_api->strnicmp 481 | #define sqlite3_unlock_notify sqlite3_api->unlock_notify 482 | #define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint 483 | #define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint 484 | #define sqlite3_wal_hook sqlite3_api->wal_hook 485 | #define sqlite3_blob_reopen sqlite3_api->blob_reopen 486 | #define sqlite3_vtab_config sqlite3_api->vtab_config 487 | #define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict 488 | /* Version 3.7.16 and later */ 489 | #define sqlite3_close_v2 sqlite3_api->close_v2 490 | #define sqlite3_db_filename sqlite3_api->db_filename 491 | #define sqlite3_db_readonly sqlite3_api->db_readonly 492 | #define sqlite3_db_release_memory sqlite3_api->db_release_memory 493 | #define sqlite3_errstr sqlite3_api->errstr 494 | #define sqlite3_stmt_busy sqlite3_api->stmt_busy 495 | #define sqlite3_stmt_readonly sqlite3_api->stmt_readonly 496 | #define sqlite3_stricmp sqlite3_api->stricmp 497 | #define sqlite3_uri_boolean sqlite3_api->uri_boolean 498 | #define sqlite3_uri_int64 sqlite3_api->uri_int64 499 | #define sqlite3_uri_parameter sqlite3_api->uri_parameter 500 | #define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf 501 | #define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 502 | /* Version 3.8.7 and later */ 503 | #define sqlite3_auto_extension sqlite3_api->auto_extension 504 | #define sqlite3_bind_blob64 sqlite3_api->bind_blob64 505 | #define sqlite3_bind_text64 sqlite3_api->bind_text64 506 | #define sqlite3_cancel_auto_extension sqlite3_api->cancel_auto_extension 507 | #define sqlite3_load_extension sqlite3_api->load_extension 508 | #define sqlite3_malloc64 sqlite3_api->malloc64 509 | #define sqlite3_msize sqlite3_api->msize 510 | #define sqlite3_realloc64 sqlite3_api->realloc64 511 | #define sqlite3_reset_auto_extension sqlite3_api->reset_auto_extension 512 | #define sqlite3_result_blob64 sqlite3_api->result_blob64 513 | #define sqlite3_result_text64 sqlite3_api->result_text64 514 | #define sqlite3_strglob sqlite3_api->strglob 515 | /* Version 3.8.11 and later */ 516 | #define sqlite3_value_dup sqlite3_api->value_dup 517 | #define sqlite3_value_free sqlite3_api->value_free 518 | #define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64 519 | #define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64 520 | /* Version 3.9.0 and later */ 521 | #define sqlite3_value_subtype sqlite3_api->value_subtype 522 | #define sqlite3_result_subtype sqlite3_api->result_subtype 523 | /* Version 3.10.0 and later */ 524 | #define sqlite3_status64 sqlite3_api->status64 525 | #define sqlite3_strlike sqlite3_api->strlike 526 | #define sqlite3_db_cacheflush sqlite3_api->db_cacheflush 527 | /* Version 3.12.0 and later */ 528 | #define sqlite3_system_errno sqlite3_api->system_errno 529 | #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ 530 | 531 | #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) 532 | /* This case when the file really is being compiled as a loadable 533 | ** extension */ 534 | # define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; 535 | # define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; 536 | # define SQLITE_EXTENSION_INIT3 \ 537 | extern const sqlite3_api_routines *sqlite3_api; 538 | #else 539 | /* This case when the file is being statically linked into the 540 | ** application */ 541 | # define SQLITE_EXTENSION_INIT1 /*no-op*/ 542 | # define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ 543 | # define SQLITE_EXTENSION_INIT3 /*no-op*/ 544 | #endif 545 | 546 | #endif /* _SQLITE3EXT_H_ */ 547 | -------------------------------------------------------------------------------- /src/utilities.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "sqlite3ext.h" 3 | #include "utilities.h" 4 | 5 | using google::protobuf::Descriptor; 6 | using google::protobuf::DescriptorPool; 7 | using google::protobuf::MessageFactory; 8 | using google::protobuf::Message; 9 | 10 | namespace sqlite_protobuf { 11 | SQLITE_EXTENSION_INIT3 12 | 13 | std::string string_from_sqlite3_value(sqlite3_value *value) 14 | { 15 | const char *text = static_cast(sqlite3_value_blob(value)); 16 | size_t text_size = static_cast(sqlite3_value_bytes(value)); 17 | return std::string(text, text_size); 18 | } 19 | 20 | namespace { 21 | 22 | bool string_equal_to_sqlite3_value(const std::string& str, sqlite3_value *val) 23 | { 24 | // Allow any conversion to take place first. 25 | const char *text = static_cast(sqlite3_value_blob(val)); 26 | 27 | size_t text_size = static_cast(sqlite3_value_bytes(val)); 28 | if (text_size != str.size()) 29 | return false; 30 | 31 | if (!text) 32 | return false; 33 | 34 | return memcmp(text, str.data(), text_size) == 0; 35 | } 36 | 37 | /* 38 | * Whenever `protobuf_load` completes loading a shared object we 39 | * need to invalidate the prototype and message caches of all threads. 40 | * This is done by having `invalidate_all_caches` increment a global 41 | * generation counter atomically, and then `get_prototype` checking 42 | * against a previously saved value in its cache. 43 | */ 44 | std::atomic global_prototype_generation; 45 | 46 | /* 47 | * We attempt to reuse `Message` objects for parsing by clearing them 48 | * to reduce memory allocation overheads. To avoid constantly 49 | * reallocating when there are really small messages mixed in with 50 | * moderately sized messages, we consider messages smaller than this 51 | * limit to actually have this size. 52 | */ 53 | const size_t MIN_MESSAGE_DATA_REUSE_SIZE = 256; 54 | 55 | struct cache { 56 | // Message type name or "". 57 | std::string message_name; 58 | 59 | // Cached prototype for the message type or `nullptr`. 60 | const Message *prototype; 61 | 62 | // Encoded message data of cached message or "". 63 | std::string message_data; 64 | 65 | // The cached result of parsing of the `message_data` or `nullptr`. 66 | std::unique_ptr message; 67 | 68 | // The maximum size of the encoded message we have parsed 69 | // using `message`. Used to reset `message` if the size of 70 | // the encoded messages drops suddenly. 71 | size_t max_message_data_size; 72 | 73 | // This is compared to the global counter before checking 74 | // for a match against `message_name`. 75 | uint64_t prototype_generation; 76 | }; 77 | 78 | inline struct cache *get_cache() { 79 | static thread_local struct cache cache; 80 | return &cache; 81 | } 82 | 83 | void invalidate_message_cache() 84 | { 85 | struct cache *cached = get_cache(); 86 | cached->message_data.clear(); 87 | cached->message.reset(); 88 | cached->max_message_data_size = 0; 89 | } 90 | 91 | void invalidate_prototype_cache() 92 | { 93 | struct cache *cached = get_cache(); 94 | cached->message_name.clear(); 95 | cached->prototype = nullptr; 96 | cached->prototype_generation = global_prototype_generation.load(std::memory_order_acquire); 97 | invalidate_message_cache(); 98 | } 99 | 100 | } // namespace 101 | 102 | void invalidate_all_caches() 103 | { 104 | global_prototype_generation.fetch_add(1, std::memory_order_acq_rel); 105 | } 106 | 107 | const Message *get_prototype(sqlite3_context *context, 108 | sqlite3_value *message_name) 109 | { 110 | struct cache *cached = get_cache(); 111 | 112 | // Look up the descriptor and prototype for the message type if necessary. 113 | uint64_t global_gen = global_prototype_generation.load(std::memory_order_acquire); 114 | if (global_gen != cached->prototype_generation || 115 | cached->prototype == nullptr || 116 | !string_equal_to_sqlite3_value(cached->message_name, message_name)) { 117 | 118 | cached->message_name = string_from_sqlite3_value(message_name); 119 | 120 | const DescriptorPool *pool = DescriptorPool::generated_pool(); 121 | const Descriptor *descriptor = pool->FindMessageTypeByName(cached->message_name); 122 | if (descriptor) { 123 | MessageFactory *factory = MessageFactory::generated_factory(); 124 | cached->prototype = factory->GetPrototype(descriptor); 125 | cached->prototype_generation = global_gen; 126 | invalidate_message_cache(); 127 | } else { 128 | sqlite3_result_error(context, "Could not find message descriptor", -1); 129 | invalidate_prototype_cache(); 130 | } 131 | } 132 | 133 | return cached->prototype; 134 | } 135 | 136 | Message *parse_message(sqlite3_context* context, 137 | sqlite3_value *message_data, 138 | sqlite3_value *message_name) 139 | { 140 | // Lookup the prototype. 141 | const Message *prototype = get_prototype(context, message_name); 142 | if (!prototype) 143 | return nullptr; 144 | 145 | struct cache *cached = get_cache(); 146 | 147 | // Parse the message if we haven't already. 148 | if (cached->message != nullptr && 149 | string_equal_to_sqlite3_value(cached->message_data,message_data)) { 150 | return cached->message.get(); 151 | } 152 | 153 | cached->message_data = string_from_sqlite3_value(message_data); 154 | 155 | // Make sure we have an empty Message object to parse with. Reuse 156 | // an existing Message object if the size of the message to parse 157 | // doesn't shrink too much. 158 | size_t cmp_size = std::max(cached->message_data.size(), 159 | MIN_MESSAGE_DATA_REUSE_SIZE); 160 | if (cached->message && cmp_size >= cached->max_message_data_size/2) { 161 | cached->message->Clear(); 162 | } else { 163 | cached->message.reset(prototype->New()); 164 | cached->max_message_data_size = 0; 165 | } 166 | 167 | cached->max_message_data_size = std::max(cached->max_message_data_size, 168 | cached->message_data.size()); 169 | 170 | if (!cached->message->ParseFromString(cached->message_data)) { 171 | sqlite3_result_error(context, "Failed to parse message", -1); 172 | invalidate_message_cache(); 173 | } 174 | 175 | return cached->message.get(); 176 | } 177 | 178 | } // namespace sqlite_protobuf 179 | -------------------------------------------------------------------------------- /src/utilities.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | #include 5 | 6 | #include "sqlite3.h" 7 | 8 | namespace sqlite_protobuf { 9 | 10 | /// Convenience method for constructing a std::string from sqlite3_value 11 | std::string string_from_sqlite3_value(sqlite3_value *value); 12 | 13 | // Looks up a prototype message for the given `message_name`. Returns 14 | // the message on success and `nullptr` on failure. The `context` is 15 | // set into an error state on failure using `sqlite3_result_error`. 16 | // 17 | // The returned message may be reused across calls in the same thread, 18 | // so should not be modified. Ownership of the message is not passed 19 | // to the caller. 20 | const google::protobuf::Message* get_prototype(sqlite3_context *context, 21 | sqlite3_value *message_name); 22 | 23 | // Parse a protobuf encoded message `message_data` of type 24 | // `message_name`. Returns a `Message` object on success and 25 | // `nullptr` on failure. The `context` is set into an error state on 26 | // failure using `sqlite3_result_error`. 27 | // 28 | // The returned message may be reused across calls in the same thread, 29 | // so should not be modified. Ownership of the message is not passed 30 | // to the caller. 31 | google::protobuf::Message *parse_message(sqlite3_context *context, 32 | sqlite3_value *message_data, 33 | sqlite3_value *message_name); 34 | 35 | // Invalidates all caches used by `get_prototype` and `parse_message`. 36 | void invalidate_all_caches(void); 37 | 38 | } // namespace sqlite_protobuf 39 | --------------------------------------------------------------------------------