├── .gitignore ├── CMakeLists.txt ├── Kconfig ├── LICENSE ├── README.rst ├── component.mk ├── config_ext.h ├── esp32.c ├── include └── sqlite3.h ├── output_console.png ├── output_shox96.png ├── private_include ├── assert.h ├── osapi.h └── shox96_0_2.h ├── shox96_0_2.c └── sqlite3.c /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .vscode 3 | 4 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(COMPONENT_ADD_INCLUDEDIRS "include") 2 | 3 | idf_component_register(SRCS "sqlite3.c" "esp32.c" "shox96_0_2.c" 4 | INCLUDE_DIRS "include" 5 | PRIV_INCLUDE_DIRS "private_include" 6 | REQUIRES mbedtls 7 | PRIV_REQUIRES console spiffs) 8 | 9 | target_compile_options(${COMPONENT_LIB} PRIVATE -std=gnu99 -g3 -fno-stack-protector -ffunction-sections -fdata-sections -fstrict-volatile-bitfields -mlongcalls -nostdlib -Wpointer-arith -Wno-error=unused-value -Wno-error=unused-label -Wno-error=unused-function -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=deprecated-declarations -Wno-error=char-subscripts -Wno-error=maybe-uninitialized -Wno-unused-parameter -Wno-sign-compare -Wno-old-style-declaration -MMD -c -DF_CPU=240000000L -DESP32 -DCORE_DEBUG_LEVEL=0 -DNDEBUG) 10 | -------------------------------------------------------------------------------- /Kconfig: -------------------------------------------------------------------------------- 1 | menu "Sqlite3" 2 | 3 | endmenu 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | SQLite3 library 2 | =============== 3 | 4 | Overview 5 | -------- 6 | 7 | Create / access SQLite3 database files using this library through SPIFFS / SD Cards. 8 | 9 | For more information, please see https://github.com/siara-cc/esp32_arduino_sqlite3_lib 10 | 11 | Installation 12 | ------------ 13 | 14 | Install ESP-IDF sdk using instructions from https://docs.espressif.com/projects/esp-idf/en/latest/get-started/. 15 | 16 | Unzip this project under esp-idf/components folder. Or if you are cloning the repository at https://github.com/espressif/esp-idf, add this repository as submodule using: 17 | 18 | `git submodule add https://github.com/siara-cc/esp32-idf-sqlite3 components/esp32-idf-sqlite3`. 19 | 20 | Using examples 21 | -------------- 22 | 23 | 1. Download [esp-idf-v4.1.1.zip](https://github.com/espressif/esp-idf/releases/download/v4.1.1/esp-idf-v4.1.1.zip) that includes submodules and unzip 24 | 2. `cd esp-idf-v4.1.1`, install using `sudo ./install.sh` 25 | 3. `git submodule add https://github.com/siara-cc/esp32-idf-sqlite3 components/esp32-idf-sqlite3` 26 | 4. `git submodule add https://github.com/siara-cc/esp32-idf-sqlite3-examples examples/esp32-idf-sqlite3-examples` 27 | 5. `cd examples/esp32-idf-sqlite3-examples/spiffs` 28 | 6. `make flash` 29 | 7. To check output, use `miniterm.py 115200` and reset board 30 | 31 | Information specific to ESP-IDF 32 | ------------------------------- 33 | 34 | To check out how this library works, please try the examples using: 35 | 36 | `git submodule add https://github.com/siara-cc/esp32-idf-sqlite3-examples examples/esp32-idf-sqlite3-examples` 37 | 38 | For instance, to check how Sqlite3 databases can be created on SPIFFS, navigate to the folder `examples/esp32-idf-sqlite3-examples/spiffs` and invoke `make flash`. 39 | 40 | Many configurations specific to SQLite library have been set in the `sdkconfig.defaults` file, such as: 41 | 42 | - Increasing stack size 43 | - Setting SPIFFS partition information 44 | - Partition size 45 | 46 | The Flash size has been assumed as 4MB for SPIFFS example. Please change any of these configurations if necessary. 47 | 48 | Issues 49 | ------ 50 | 51 | If you face any issues, please contact the author (Arundale Ramanathan) at arun@siara.cc or create a github issue. 52 | -------------------------------------------------------------------------------- /component.mk: -------------------------------------------------------------------------------- 1 | # 2 | # Component Makefile 3 | # 4 | COMPONENT_PRIV_INCLUDEDIRS=private_include 5 | CFLAGS = -std=gnu99 -g3 -fno-stack-protector -ffunction-sections -fdata-sections -fstrict-volatile-bitfields -mlongcalls -nostdlib -Wpointer-arith -Wno-error=unused-value -Wno-error=unused-label -Wno-error=unused-function -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=deprecated-declarations -Wno-unused-parameter -Wno-sign-compare -Wno-old-style-declaration -MMD -c -DF_CPU=240000000L -DESP32 -DCORE_DEBUG_LEVEL=0 -DNDEBUG 6 | 7 | -------------------------------------------------------------------------------- /config_ext.h: -------------------------------------------------------------------------------- 1 | #define BUILD_sqlite -DNDEBUG 2 | #define SQLITE_CORE 1 3 | #define SQLITE_NO_SYNC 1 4 | #define YYSTACKDEPTH 20 5 | #define SQLITE_TEMP_STORE 1 6 | #define SQLITE_SYSTEM_MALLOC 1 7 | #define SQLITE_OS_OTHER 1 8 | #define SQLITE_THREADSAFE 0 9 | #define SQLITE_MUTEX_APPDEF 1 10 | #define SQLITE_SECURE_DELETE 0 11 | #define SQLITE_SMALL_STACK 1 12 | #define SQLITE_DISABLE_LFS 1 13 | #define SQLITE_DISABLE_DIRSYNC 1 14 | #define SQLITE_DISABLE_FTS3_UNICODE 1 15 | #define SQLITE_DISABLE_FTS4_DEFERRED 1 16 | #define SQLITE_LIKE_DOESNT_MATCH_BLOBS 1 17 | #define SQLITE_DEFAULT_CACHE_SIZE -1 18 | #define SQLITE_DEFAULT_FOREIGN_KEYS 0 19 | #define SQLITE_DEFAULT_MEMSTATUS 0 20 | #define SQLITE_DEFAULT_MMAP_SIZE 0 21 | #define SQLITE_DEFAULT_LOCKING_MODE 1 22 | #define SQLITE_DEFAULT_LOOKASIDE 512,128 23 | #define SQLITE_DEFAULT_PAGE_SIZE 512 24 | #define SQLITE_DEFAULT_PCACHE_INITSZ 8 25 | #define SQLITE_MAX_DEFAULT_PAGE_SIZE 32768 26 | #define SQLITE_POWERSAFE_OVERWRITE 1 27 | #define SQLITE_SORTER_PMASZ 4 28 | #define SQLITE_MAX_EXPR_DEPTH 0 29 | #undef SQLITE_OMIT_ALTERTABLE 30 | #undef SQLITE_OMIT_ANALYZE 31 | #undef SQLITE_OMIT_ATTACH 32 | #define SQLITE_OMIT_AUTHORIZATION 1 33 | #undef SQLITE_OMIT_AUTOINCREMENT 34 | #define SQLITE_OMIT_AUTOINIT 1 35 | #define SQLITE_OMIT_AUTOMATIC_INDEX 1 36 | #define SQLITE_OMIT_AUTORESET 1 37 | #define SQLITE_OMIT_AUTOVACUUM 1 38 | #undef SQLITE_OMIT_BETWEEN_OPTIMIZATION 39 | #define SQLITE_OMIT_BLOB_LITERAL 1 40 | #define SQLITE_OMIT_BTREECOUNT 1 41 | #define SQLITE_OMIT_BUILTIN_TEST 1 42 | #define SQLITE_OMIT_CAST 1 43 | #define SQLITE_OMIT_CHECK 1 44 | #define SQLITE_OMIT_COMPILEOPTION_DIAGS 1 45 | #define SQLITE_OMIT_COMPOUND_SELECT 1 46 | #define SQLITE_OMIT_CONFLICT_CLAUSE 1 47 | #undef SQLITE_OMIT_CTE 48 | #define SQLITE_OMIT_DECLTYPE 1 49 | #define SQLITE_OMIT_DEPRECATED 1 50 | #undef SQLITE_OMIT_DISKIO 51 | #define SQLITE_OMIT_EXPLAIN 1 52 | #define SQLITE_OMIT_FLAG_PRAGMAS 1 53 | #define SQLITE_OMIT_FOREIGN_KEY 1 54 | #define SQLITE_OMIT_GET_TABLE 1 55 | #define SQLITE_OMIT_INCRBLOB 1 56 | #define SQLITE_OMIT_INTEGRITY_CHECK 1 57 | #undef SQLITE_OMIT_LIKE_OPTIMIZATION 58 | #define SQLITE_OMIT_LOAD_EXTENSION 1 59 | #undef SQLITE_OMIT_LOCALTIME 60 | #define SQLITE_OMIT_LOOKASIDE 1 61 | #undef SQLITE_OMIT_MEMORYDB 62 | #undef SQLITE_OMIT_OR_OPTIMIZATION 63 | #undef SQLITE_OMIT_PAGER_PRAGMAS 64 | #define SQLITE_OMIT_PARSER_TRACE 1 65 | #undef SQLITE_OMIT_PRAGMA 66 | #define SQLITE_OMIT_PROGRESS_CALLBACK 1 67 | #define SQLITE_OMIT_QUICKBALANCE 1 68 | #undef SQLITE_OMIT_REINDEX 69 | #define SQLITE_OMIT_SCHEMA_PRAGMAS 1 70 | #define SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS 1 71 | #define SQLITE_OMIT_SHARED_CACHE 1 72 | #define SQLITE_OMIT_TCL_VARIABLE 1 73 | #define SQLITE_OMIT_TEMPDB 1 74 | #define SQLITE_OMIT_TRACE 1 75 | #undef SQLITE_OMIT_TRIGGER 76 | #define SQLITE_OMIT_TRUNCATE_OPTIMIZATION 1 77 | #define SQLITE_OMIT_UTF16 1 78 | #undef SQLITE_OMIT_VACUUM 79 | #undef SQLITE_OMIT_VIEW 80 | #undef SQLITE_OMIT_VIRTUALTABLE 81 | #define SQLITE_OMIT_WAL 1 82 | #undef SQLITE_OMIT_WSD 83 | #define SQLITE_OMIT_XFER_OPT 1 84 | #define SQLITE_PERFORMANCE_TRACE 1 85 | /* #define SQLITE_OMIT_COMPLETE 1 */ 86 | /* #define SQLITE_OMIT_SUBQUERY 1 */ 87 | /* #define SQLITE_OMIT_DATETIME_FUNCS 1 */ 88 | /* #define SQLITE_OMIT_FLOATING_POINT 1 */ 89 | #define SQLITE_COUNTOFVIEW_OPTIMIZATION 0 90 | -------------------------------------------------------------------------------- /esp32.c: -------------------------------------------------------------------------------- 1 | /* From: https://chromium.googlesource.com/chromium/src.git/+/4.1.249.1050/third_party/sqlite/src/os_symbian.cc 2 | * https://github.com/spsoft/spmemvfs/tree/master/spmemvfs 3 | * http://www.sqlite.org/src/doc/trunk/src/test_demovfs.c 4 | * http://www.sqlite.org/src/doc/trunk/src/test_vfstrace.c 5 | * http://www.sqlite.org/src/doc/trunk/src/test_onefile.c 6 | * http://www.sqlite.org/src/doc/trunk/src/test_vfs.c 7 | * https://github.com/nodemcu/nodemcu-firmware/blob/master/app/sqlite3/esp8266.c 8 | **/ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "shox96_0_2.h" 23 | 24 | #undef dbg_printf 25 | //#define dbg_printf(...) printf(__VA_ARGS__) 26 | #define dbg_printf(...) 27 | #define CACHEBLOCKSZ 64 28 | #define esp32_DEFAULT_MAXNAMESIZE 100 29 | 30 | // From https://stackoverflow.com/questions/19758270/read-varint-from-linux-sockets#19760246 31 | // Encode an unsigned 64-bit varint. Returns number of encoded bytes. 32 | // 'buffer' must have room for up to 10 bytes. 33 | int encode_unsigned_varint(uint8_t *buffer, uint64_t value) { 34 | int encoded = 0; 35 | do { 36 | uint8_t next_byte = value & 0x7F; 37 | value >>= 7; 38 | if (value) 39 | next_byte |= 0x80; 40 | buffer[encoded++] = next_byte; 41 | } while (value); 42 | return encoded; 43 | } 44 | 45 | uint64_t decode_unsigned_varint(const uint8_t *data, int *decoded_bytes) { 46 | int i = 0; 47 | uint64_t decoded_value = 0; 48 | int shift_amount = 0; 49 | do { 50 | decoded_value |= (uint64_t)(data[i] & 0x7F) << shift_amount; 51 | shift_amount += 7; 52 | } while ((data[i++] & 0x80) != 0); 53 | *decoded_bytes = i; 54 | return decoded_value; 55 | } 56 | 57 | int esp32_Close(sqlite3_file*); 58 | int esp32_Lock(sqlite3_file *, int); 59 | int esp32_Unlock(sqlite3_file*, int); 60 | int esp32_Sync(sqlite3_file*, int); 61 | int esp32_Open(sqlite3_vfs*, const char *, sqlite3_file *, int, int*); 62 | int esp32_Read(sqlite3_file*, void*, int, sqlite3_int64); 63 | int esp32_Write(sqlite3_file*, const void*, int, sqlite3_int64); 64 | int esp32_Truncate(sqlite3_file*, sqlite3_int64); 65 | int esp32_Delete(sqlite3_vfs*, const char *, int); 66 | int esp32_FileSize(sqlite3_file*, sqlite3_int64*); 67 | int esp32_Access(sqlite3_vfs*, const char*, int, int*); 68 | int esp32_FullPathname( sqlite3_vfs*, const char *, int, char*); 69 | int esp32_CheckReservedLock(sqlite3_file*, int *); 70 | int esp32_FileControl(sqlite3_file *, int, void*); 71 | int esp32_SectorSize(sqlite3_file*); 72 | int esp32_DeviceCharacteristics(sqlite3_file*); 73 | void* esp32_DlOpen(sqlite3_vfs*, const char *); 74 | void esp32_DlError(sqlite3_vfs*, int, char*); 75 | void (*esp32_DlSym (sqlite3_vfs*, void*, const char*))(void); 76 | void esp32_DlClose(sqlite3_vfs*, void*); 77 | int esp32_Randomness(sqlite3_vfs*, int, char*); 78 | int esp32_Sleep(sqlite3_vfs*, int); 79 | int esp32_CurrentTime(sqlite3_vfs*, double*); 80 | 81 | int esp32mem_Close(sqlite3_file*); 82 | int esp32mem_Read(sqlite3_file*, void*, int, sqlite3_int64); 83 | int esp32mem_Write(sqlite3_file*, const void*, int, sqlite3_int64); 84 | int esp32mem_FileSize(sqlite3_file*, sqlite3_int64*); 85 | int esp32mem_Sync(sqlite3_file*, int); 86 | 87 | typedef struct st_linkedlist { 88 | uint16_t blockid; 89 | struct st_linkedlist *next; 90 | uint8_t data[CACHEBLOCKSZ]; 91 | } linkedlist_t, *pLinkedList_t; 92 | 93 | typedef struct st_filecache { 94 | uint32_t size; 95 | linkedlist_t *list; 96 | } filecache_t, *pFileCache_t; 97 | 98 | typedef struct esp32_file { 99 | sqlite3_file base; 100 | FILE *fd; 101 | filecache_t *cache; 102 | char name[esp32_DEFAULT_MAXNAMESIZE]; 103 | } esp32_file; 104 | 105 | sqlite3_vfs esp32Vfs = { 106 | 1, // iVersion 107 | sizeof(esp32_file), // szOsFile 108 | 101, // mxPathname 109 | NULL, // pNext 110 | "esp32", // name 111 | 0, // pAppData 112 | esp32_Open, // xOpen 113 | esp32_Delete, // xDelete 114 | esp32_Access, // xAccess 115 | esp32_FullPathname, // xFullPathname 116 | esp32_DlOpen, // xDlOpen 117 | esp32_DlError, // xDlError 118 | esp32_DlSym, // xDlSym 119 | esp32_DlClose, // xDlClose 120 | esp32_Randomness, // xRandomness 121 | esp32_Sleep, // xSleep 122 | esp32_CurrentTime, // xCurrentTime 123 | 0 // xGetLastError 124 | }; 125 | 126 | const sqlite3_io_methods esp32IoMethods = { 127 | 1, 128 | esp32_Close, 129 | esp32_Read, 130 | esp32_Write, 131 | esp32_Truncate, 132 | esp32_Sync, 133 | esp32_FileSize, 134 | esp32_Lock, 135 | esp32_Unlock, 136 | esp32_CheckReservedLock, 137 | esp32_FileControl, 138 | esp32_SectorSize, 139 | esp32_DeviceCharacteristics 140 | }; 141 | 142 | const sqlite3_io_methods esp32MemMethods = { 143 | 1, 144 | esp32mem_Close, 145 | esp32mem_Read, 146 | esp32mem_Write, 147 | esp32_Truncate, 148 | esp32mem_Sync, 149 | esp32mem_FileSize, 150 | esp32_Lock, 151 | esp32_Unlock, 152 | esp32_CheckReservedLock, 153 | esp32_FileControl, 154 | esp32_SectorSize, 155 | esp32_DeviceCharacteristics 156 | }; 157 | 158 | uint32_t linkedlist_store (linkedlist_t **leaf, uint32_t offset, uint32_t len, const uint8_t *data) { 159 | const uint8_t blank[CACHEBLOCKSZ] = { 0 }; 160 | uint16_t blockid = offset/CACHEBLOCKSZ; 161 | linkedlist_t *block; 162 | 163 | if (!memcmp(data, blank, CACHEBLOCKSZ)) 164 | return len; 165 | 166 | block = *leaf; 167 | if (!block || ( block->blockid != blockid ) ) { 168 | block = (linkedlist_t *) sqlite3_malloc ( sizeof( linkedlist_t ) ); 169 | if (!block) 170 | return SQLITE_NOMEM; 171 | 172 | memset (block->data, 0, CACHEBLOCKSZ); 173 | block->blockid = blockid; 174 | } 175 | 176 | if (!*leaf) { 177 | *leaf = block; 178 | block->next = NULL; 179 | } else if (block != *leaf) { 180 | if (block->blockid > (*leaf)->blockid) { 181 | block->next = (*leaf)->next; 182 | (*leaf)->next = block; 183 | } else { 184 | block->next = (*leaf); 185 | (*leaf) = block; 186 | } 187 | } 188 | 189 | memcpy (block->data + offset%CACHEBLOCKSZ, data, len); 190 | 191 | return len; 192 | } 193 | 194 | uint32_t filecache_pull (pFileCache_t cache, uint32_t offset, uint32_t len, uint8_t *data) { 195 | uint16_t i; 196 | float blocks; 197 | uint32_t r = 0; 198 | 199 | blocks = ( offset % CACHEBLOCKSZ + len ) / (float) CACHEBLOCKSZ; 200 | if (blocks == 0.0) 201 | return 0; 202 | if (!cache->list) 203 | return 0; 204 | 205 | if (( blocks - (int) blocks) > 0.0) 206 | blocks = blocks + 1.0; 207 | 208 | for (i = 0; i < (uint16_t) blocks; i++) { 209 | uint16_t round; 210 | float relablock; 211 | linkedlist_t *leaf; 212 | uint32_t relaoffset, relalen; 213 | uint8_t * reladata = (uint8_t*) data; 214 | 215 | relalen = len - r; 216 | 217 | reladata = reladata + r; 218 | relaoffset = offset + r; 219 | 220 | round = CACHEBLOCKSZ - relaoffset%CACHEBLOCKSZ; 221 | if (relalen > round) relalen = round; 222 | 223 | for (leaf = cache->list; leaf && leaf->next; leaf = leaf->next) { 224 | if ( ( leaf->next->blockid * CACHEBLOCKSZ ) > relaoffset ) 225 | break; 226 | } 227 | 228 | relablock = relaoffset/((float)CACHEBLOCKSZ) - leaf->blockid; 229 | 230 | if ( ( relablock >= 0 ) && ( relablock < 1 ) ) 231 | memcpy (data + r, leaf->data + (relaoffset % CACHEBLOCKSZ), relalen); 232 | 233 | r = r + relalen; 234 | } 235 | 236 | return 0; 237 | } 238 | 239 | uint32_t filecache_push (pFileCache_t cache, uint32_t offset, uint32_t len, const uint8_t *data) { 240 | uint16_t i; 241 | float blocks; 242 | uint32_t r = 0; 243 | uint8_t updateroot = 0x1; 244 | 245 | blocks = ( offset % CACHEBLOCKSZ + len ) / (float) CACHEBLOCKSZ; 246 | 247 | if (blocks == 0.0) 248 | return 0; 249 | 250 | if (( blocks - (int) blocks) > 0.0) 251 | blocks = blocks + 1.0; 252 | 253 | for (i = 0; i < (uint16_t) blocks; i++) { 254 | uint16_t round; 255 | uint32_t localr; 256 | linkedlist_t *leaf; 257 | uint32_t relaoffset, relalen; 258 | uint8_t * reladata = (uint8_t*) data; 259 | 260 | relalen = len - r; 261 | 262 | reladata = reladata + r; 263 | relaoffset = offset + r; 264 | 265 | round = CACHEBLOCKSZ - relaoffset%CACHEBLOCKSZ; 266 | if (relalen > round) relalen = round; 267 | 268 | for (leaf = cache->list; leaf && leaf->next; leaf = leaf->next) { 269 | if ( ( leaf->next->blockid * CACHEBLOCKSZ ) > relaoffset ) 270 | break; 271 | updateroot = 0x0; 272 | } 273 | 274 | localr = linkedlist_store(&leaf, relaoffset, (relalen > CACHEBLOCKSZ) ? CACHEBLOCKSZ : relalen, reladata); 275 | if (localr == SQLITE_NOMEM) 276 | return SQLITE_NOMEM; 277 | 278 | r = r + localr; 279 | 280 | if (updateroot & 0x1) 281 | cache->list = leaf; 282 | } 283 | 284 | if (offset + len > cache->size) 285 | cache->size = offset + len; 286 | 287 | return r; 288 | } 289 | 290 | void filecache_free (pFileCache_t cache) { 291 | pLinkedList_t ll = cache->list, next; 292 | 293 | while (ll != NULL) { 294 | next = ll->next; 295 | sqlite3_free (ll); 296 | ll = next; 297 | } 298 | } 299 | 300 | int esp32mem_Close(sqlite3_file *id) 301 | { 302 | esp32_file *file = (esp32_file*) id; 303 | 304 | filecache_free(file->cache); 305 | sqlite3_free (file->cache); 306 | 307 | dbg_printf("esp32mem_Close: %s OK\n", file->name); 308 | return SQLITE_OK; 309 | } 310 | 311 | int esp32mem_Read(sqlite3_file *id, void *buffer, int amount, sqlite3_int64 offset) 312 | { 313 | int32_t ofst; 314 | esp32_file *file = (esp32_file*) id; 315 | ofst = (int32_t)(offset & 0x7FFFFFFF); 316 | 317 | filecache_pull (file->cache, ofst, amount, (uint8_t *) buffer); 318 | 319 | dbg_printf("esp32mem_Read: %s [%d] [%d] OK\n", file->name, ofst, amount); 320 | return SQLITE_OK; 321 | } 322 | 323 | int esp32mem_Write(sqlite3_file *id, const void *buffer, int amount, sqlite3_int64 offset) 324 | { 325 | int32_t ofst; 326 | esp32_file *file = (esp32_file*) id; 327 | 328 | ofst = (int32_t)(offset & 0x7FFFFFFF); 329 | 330 | filecache_push (file->cache, ofst, amount, (const uint8_t *) buffer); 331 | 332 | dbg_printf("esp32mem_Write: %s [%d] [%d] OK\n", file->name, ofst, amount); 333 | return SQLITE_OK; 334 | } 335 | 336 | int esp32mem_Sync(sqlite3_file *id, int flags) 337 | { 338 | esp32_file *file = (esp32_file*) id; 339 | dbg_printf("esp32mem_Sync: %s OK\n", file->name); 340 | return SQLITE_OK; 341 | } 342 | 343 | int esp32mem_FileSize(sqlite3_file *id, sqlite3_int64 *size) 344 | { 345 | esp32_file *file = (esp32_file*) id; 346 | 347 | *size = 0LL | file->cache->size; 348 | dbg_printf("esp32mem_FileSize: %s [%d] OK\n", file->name, file->cache->size); 349 | return SQLITE_OK; 350 | } 351 | 352 | int esp32_Open( sqlite3_vfs * vfs, const char * path, sqlite3_file * file, int flags, int * outflags ) 353 | { 354 | int rc; 355 | char mode[5]; 356 | esp32_file *p = (esp32_file*) file; 357 | 358 | strcpy(mode, "r"); 359 | if ( path == NULL ) return SQLITE_IOERR; 360 | dbg_printf("esp32_Open: 0o %s %s\n", path, mode); 361 | if( flags&SQLITE_OPEN_READONLY ) 362 | strcpy(mode, "r"); 363 | if( flags&SQLITE_OPEN_READWRITE || flags&SQLITE_OPEN_MAIN_JOURNAL ) { 364 | int result; 365 | if (SQLITE_OK != esp32_Access(vfs, path, flags, &result)) 366 | return SQLITE_CANTOPEN; 367 | 368 | if (result == 1) 369 | strcpy(mode, "r+"); 370 | else 371 | strcpy(mode, "w+"); 372 | } 373 | 374 | dbg_printf("esp32_Open: 1o %s %s\n", path, mode); 375 | memset (p, 0, sizeof(esp32_file)); 376 | 377 | strncpy (p->name, path, esp32_DEFAULT_MAXNAMESIZE); 378 | p->name[esp32_DEFAULT_MAXNAMESIZE-1] = '\0'; 379 | 380 | if( flags&SQLITE_OPEN_MAIN_JOURNAL ) { 381 | p->fd = 0; 382 | p->cache = (filecache_t *) sqlite3_malloc(sizeof (filecache_t)); 383 | if (! p->cache ) 384 | return SQLITE_NOMEM; 385 | memset (p->cache, 0, sizeof(filecache_t)); 386 | 387 | p->base.pMethods = &esp32MemMethods; 388 | dbg_printf("esp32_Open: 2o %s MEM OK\n", p->name); 389 | return SQLITE_OK; 390 | } 391 | 392 | p->fd = fopen(path, mode); 393 | if ( p->fd <= 0 ) { 394 | return SQLITE_CANTOPEN; 395 | } 396 | 397 | p->base.pMethods = &esp32IoMethods; 398 | dbg_printf("esp32_Open: 2o %s OK\n", p->name); 399 | return SQLITE_OK; 400 | } 401 | 402 | int esp32_Close(sqlite3_file *id) 403 | { 404 | esp32_file *file = (esp32_file*) id; 405 | 406 | int rc = fclose(file->fd); 407 | dbg_printf("esp32_Close: %s %d\n", file->name, rc); 408 | return rc ? SQLITE_IOERR_CLOSE : SQLITE_OK; 409 | } 410 | 411 | int esp32_Read(sqlite3_file *id, void *buffer, int amount, sqlite3_int64 offset) 412 | { 413 | size_t nRead; 414 | int32_t ofst, iofst; 415 | esp32_file *file = (esp32_file*) id; 416 | 417 | iofst = (int32_t)(offset & 0x7FFFFFFF); 418 | 419 | dbg_printf("esp32_Read: 1r %s %d %lld[%d] \n", file->name, amount, offset, iofst); 420 | ofst = fseek(file->fd, iofst, SEEK_SET); 421 | if (ofst != 0) { 422 | dbg_printf("esp32_Read: 2r %d != %d FAIL\n", ofst, iofst); 423 | return SQLITE_IOERR_SHORT_READ /* SQLITE_IOERR_SEEK */; 424 | } 425 | 426 | nRead = fread(buffer, 1, amount, file->fd); 427 | if ( nRead == amount ) { 428 | dbg_printf("esp32_Read: 3r %s %u %d OK\n", file->name, nRead, amount); 429 | return SQLITE_OK; 430 | } else if ( nRead >= 0 ) { 431 | dbg_printf("esp32_Read: 3r %s %u %d FAIL\n", file->name, nRead, amount); 432 | return SQLITE_IOERR_SHORT_READ; 433 | } 434 | 435 | dbg_printf("esp32_Read: 4r %s FAIL\n", file->name); 436 | return SQLITE_IOERR_READ; 437 | } 438 | 439 | int esp32_Write(sqlite3_file *id, const void *buffer, int amount, sqlite3_int64 offset) 440 | { 441 | size_t nWrite; 442 | int32_t ofst, iofst; 443 | esp32_file *file = (esp32_file*) id; 444 | 445 | iofst = (int32_t)(offset & 0x7FFFFFFF); 446 | 447 | dbg_printf("esp32_Write: 1w %s %d %lld[%d] \n", file->name, amount, offset, iofst); 448 | ofst = fseek(file->fd, iofst, SEEK_SET); 449 | if (ofst != 0) { 450 | return SQLITE_IOERR_SEEK; 451 | } 452 | 453 | nWrite = fwrite(buffer, 1, amount, file->fd); 454 | if ( nWrite != amount ) { 455 | dbg_printf("esp32_Write: 2w %s %u %d\n", file->name, nWrite, amount); 456 | return SQLITE_IOERR_WRITE; 457 | } 458 | 459 | dbg_printf("esp32_Write: 3w %s OK\n", file->name); 460 | return SQLITE_OK; 461 | } 462 | 463 | int esp32_Truncate(sqlite3_file *id, sqlite3_int64 bytes) 464 | { 465 | esp32_file *file = (esp32_file*) id; 466 | //int fno = fileno(file->fd); 467 | //if (fno == -1) 468 | // return SQLITE_IOERR_TRUNCATE; 469 | //if (ftruncate(fno, 0)) 470 | // return SQLITE_IOERR_TRUNCATE; 471 | 472 | dbg_printf("esp32_Truncate:\n"); 473 | return SQLITE_OK; 474 | } 475 | 476 | int esp32_Delete( sqlite3_vfs * vfs, const char * path, int syncDir ) 477 | { 478 | int32_t rc = remove( path ); 479 | if (rc) 480 | return SQLITE_IOERR_DELETE; 481 | 482 | dbg_printf("esp32_Delete: %s OK\n", path); 483 | return SQLITE_OK; 484 | } 485 | 486 | int esp32_FileSize(sqlite3_file *id, sqlite3_int64 *size) 487 | { 488 | esp32_file *file = (esp32_file*) id; 489 | dbg_printf("esp32_FileSize: %s: ", file->name); 490 | struct stat st; 491 | int fno = fileno(file->fd); 492 | if (fno == -1) 493 | return SQLITE_IOERR_FSTAT; 494 | if (fstat(fno, &st)) 495 | return SQLITE_IOERR_FSTAT; 496 | *size = st.st_size; 497 | dbg_printf(" %ld[%lld]\n", st.st_size, *size); 498 | return SQLITE_OK; 499 | } 500 | 501 | int esp32_Sync(sqlite3_file *id, int flags) 502 | { 503 | esp32_file *file = (esp32_file*) id; 504 | 505 | int rc = fflush( file->fd ); 506 | fsync(fileno(file->fd)); 507 | dbg_printf("esp32_Sync( %s: ): %d \n",file->name, rc); 508 | 509 | return rc ? SQLITE_IOERR_FSYNC : SQLITE_OK; 510 | } 511 | 512 | int esp32_Access( sqlite3_vfs * vfs, const char * path, int flags, int * result ) 513 | { 514 | struct stat st; 515 | memset(&st, 0, sizeof(struct stat)); 516 | int rc = stat( path, &st ); 517 | *result = ( rc != -1 ); 518 | 519 | dbg_printf("esp32_Access: %s %d %d %ld\n", path, *result, rc, st.st_size); 520 | return SQLITE_OK; 521 | } 522 | 523 | int esp32_FullPathname( sqlite3_vfs * vfs, const char * path, int len, char * fullpath ) 524 | { 525 | //structure stat does not have name. 526 | //struct stat st; 527 | //int32_t rc = stat( path, &st ); 528 | //if ( rc == 0 ){ 529 | // strncpy( fullpath, st.name, len ); 530 | //} else { 531 | // strncpy( fullpath, path, len ); 532 | //} 533 | 534 | // As now just copy the path 535 | strncpy( fullpath, path, len ); 536 | fullpath[ len - 1 ] = '\0'; 537 | 538 | dbg_printf("esp32_FullPathname: %s\n", fullpath); 539 | return SQLITE_OK; 540 | } 541 | 542 | int esp32_Lock(sqlite3_file *id, int lock_type) 543 | { 544 | esp32_file *file = (esp32_file*) id; 545 | 546 | dbg_printf("esp32_Lock:Not locked\n"); 547 | return SQLITE_OK; 548 | } 549 | 550 | int esp32_Unlock(sqlite3_file *id, int lock_type) 551 | { 552 | esp32_file *file = (esp32_file*) id; 553 | 554 | dbg_printf("esp32_Unlock:\n"); 555 | return SQLITE_OK; 556 | } 557 | 558 | int esp32_CheckReservedLock(sqlite3_file *id, int *result) 559 | { 560 | esp32_file *file = (esp32_file*) id; 561 | 562 | *result = 0; 563 | 564 | dbg_printf("esp32_CheckReservedLock:\n"); 565 | return SQLITE_OK; 566 | } 567 | 568 | int esp32_FileControl(sqlite3_file *id, int op, void *arg) 569 | { 570 | esp32_file *file = (esp32_file*) id; 571 | 572 | dbg_printf("esp32_FileControl:\n"); 573 | return SQLITE_OK; 574 | } 575 | 576 | int esp32_SectorSize(sqlite3_file *id) 577 | { 578 | esp32_file *file = (esp32_file*) id; 579 | 580 | dbg_printf("esp32_SectorSize:\n"); 581 | return SPI_FLASH_SEC_SIZE; 582 | } 583 | 584 | int esp32_DeviceCharacteristics(sqlite3_file *id) 585 | { 586 | esp32_file *file = (esp32_file*) id; 587 | 588 | dbg_printf("esp32_DeviceCharacteristics:\n"); 589 | return 0; 590 | } 591 | 592 | void * esp32_DlOpen( sqlite3_vfs * vfs, const char * path ) 593 | { 594 | dbg_printf("esp32_DlOpen:\n"); 595 | return NULL; 596 | } 597 | 598 | void esp32_DlError( sqlite3_vfs * vfs, int len, char * errmsg ) 599 | { 600 | dbg_printf("esp32_DlError:\n"); 601 | return; 602 | } 603 | 604 | void ( * esp32_DlSym ( sqlite3_vfs * vfs, void * handle, const char * symbol ) ) ( void ) 605 | { 606 | dbg_printf("esp32_DlSym:\n"); 607 | return NULL; 608 | } 609 | 610 | void esp32_DlClose( sqlite3_vfs * vfs, void * handle ) 611 | { 612 | dbg_printf("esp32_DlClose:\n"); 613 | return; 614 | } 615 | 616 | int esp32_Randomness( sqlite3_vfs * vfs, int len, char * buffer ) 617 | { 618 | long rdm; 619 | int sz = 1 + (len / sizeof(long)); 620 | char a_rdm[sz * sizeof(long)]; 621 | while (sz--) { 622 | rdm = esp_random(); 623 | memcpy(a_rdm + sz * sizeof(long), &rdm, sizeof(long)); 624 | } 625 | memcpy(buffer, a_rdm, len); 626 | dbg_printf("esp32_Randomness\n"); 627 | return SQLITE_OK; 628 | } 629 | 630 | int esp32_Sleep( sqlite3_vfs * vfs, int microseconds ) 631 | { 632 | ets_delay_us(microseconds); 633 | dbg_printf("esp32_Sleep:\n"); 634 | return SQLITE_OK; 635 | } 636 | 637 | int esp32_CurrentTime( sqlite3_vfs * vfs, double * result ) 638 | { 639 | time_t t = time(NULL); 640 | *result = t / 86400.0 + 2440587.5; 641 | // This is stubbed out until we have a working RTCTIME solution; 642 | // as it stood, this would always have returned the UNIX epoch. 643 | //*result = 2440587.5; 644 | dbg_printf("esp32_CurrentTime: %g\n", *result); 645 | return SQLITE_OK; 646 | } 647 | 648 | static void shox96_0_2c(sqlite3_context *context, int argc, sqlite3_value **argv) { 649 | int nIn, nOut; 650 | long int nOut2; 651 | const unsigned char *inBuf; 652 | unsigned char *outBuf; 653 | unsigned char vInt[9]; 654 | int vIntLen; 655 | 656 | assert( argc==1 ); 657 | nIn = sqlite3_value_bytes(argv[0]); 658 | inBuf = (unsigned char *) sqlite3_value_blob(argv[0]); 659 | nOut = 13 + nIn + (nIn+999)/1000; 660 | vIntLen = encode_unsigned_varint(vInt, (uint64_t) nIn); 661 | 662 | outBuf = (unsigned char *) malloc( nOut+vIntLen ); 663 | memcpy(outBuf, vInt, vIntLen); 664 | nOut2 = shox96_0_2_compress((const char *) inBuf, nIn, (char *) &outBuf[vIntLen], NULL); 665 | sqlite3_result_blob(context, outBuf, nOut2+vIntLen, free); 666 | } 667 | 668 | static void shox96_0_2d(sqlite3_context *context, int argc, sqlite3_value **argv) { 669 | unsigned int nIn, nOut, rc; 670 | const unsigned char *inBuf; 671 | unsigned char *outBuf; 672 | long int nOut2; 673 | uint64_t inBufLen64; 674 | int vIntLen; 675 | 676 | assert( argc==1 ); 677 | 678 | if (sqlite3_value_type(argv[0]) != SQLITE_BLOB) 679 | return; 680 | 681 | nIn = sqlite3_value_bytes(argv[0]); 682 | if (nIn < 2){ 683 | return; 684 | } 685 | inBuf = (unsigned char *) sqlite3_value_blob(argv[0]); 686 | inBufLen64 = decode_unsigned_varint(inBuf, &vIntLen); 687 | nOut = (unsigned int) inBufLen64; 688 | outBuf = (unsigned char *) malloc( nOut ); 689 | //nOut2 = (long int)nOut; 690 | nOut2 = shox96_0_2_decompress((const char *) (inBuf + vIntLen), nIn - vIntLen, (char *) outBuf, NULL); 691 | //if( rc!=Z_OK ){ 692 | // free(outBuf); 693 | //}else{ 694 | sqlite3_result_blob(context, outBuf, nOut2, free); 695 | //} 696 | } 697 | 698 | int registerShox96_0_2(sqlite3 *db, const char **pzErrMsg, const struct sqlite3_api_routines *pThunk) { 699 | sqlite3_create_function(db, "shox96_0_2c", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, shox96_0_2c, 0, 0); 700 | sqlite3_create_function(db, "shox96_0_2d", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, shox96_0_2d, 0, 0); 701 | return SQLITE_OK; 702 | } 703 | 704 | int sqlite3_os_init(void){ 705 | sqlite3_vfs_register(&esp32Vfs, 1); 706 | sqlite3_auto_extension((void (*)())registerShox96_0_2); 707 | return SQLITE_OK; 708 | } 709 | 710 | int sqlite3_os_end(void){ 711 | return SQLITE_OK; 712 | } 713 | -------------------------------------------------------------------------------- /output_console.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32-idf-sqlite3/9752258e0a8166417804b26cb9c458e7cad9c05a/output_console.png -------------------------------------------------------------------------------- /output_shox96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32-idf-sqlite3/9752258e0a8166417804b26cb9c458e7cad9c05a/output_shox96.png -------------------------------------------------------------------------------- /private_include/assert.h: -------------------------------------------------------------------------------- 1 | #ifndef __DUMMY_ASSERT_H 2 | #define __DUMMY_ASSERT_H 3 | #ifdef NDEBUG 4 | #define assert(x) 5 | #else 6 | #include_next 7 | #endif 8 | #endif 9 | 10 | -------------------------------------------------------------------------------- /private_include/osapi.h: -------------------------------------------------------------------------------- 1 | #ifndef __OSAPI_H__ 2 | #define __OSAPI_H__ 3 | 4 | #define os_malloc(x) malloc(x) 5 | #define os_realloc(x, y) realloc(x, y) 6 | #define os_free(x) free(x) 7 | 8 | #endif 9 | 10 | -------------------------------------------------------------------------------- /private_include/shox96_0_2.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2019 Siara Logics (cc) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * @author Arundale R. 17 | * 18 | */ 19 | #ifndef shox96_0_2_0 20 | #define shox96_0_2_0 21 | 22 | struct lnk_lst { 23 | char *data; 24 | struct lnk_lst *previous; 25 | }; 26 | 27 | extern int shox96_0_2_compress(const char *in, int len, char *out, struct lnk_lst *prev_lines); 28 | extern int shox96_0_2_decompress(const char *in, int len, char *out, struct lnk_lst *prev_lines); 29 | #endif 30 | -------------------------------------------------------------------------------- /shox96_0_2.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2019 Siara Logics (cc) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * @author Arundale R. 17 | * 18 | */ 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "shox96_0_2.h" 26 | 27 | typedef unsigned char byte; 28 | 29 | unsigned int c_95[95] = {16384, 16256, 15744, 16192, 15328, 15344, 15360, 16064, 15264, 15296, 15712, 15200, 14976, 15040, 14848, 15104, 14528, 14592, 14656, 14688, 14720, 14752, 14784, 14816, 14832, 14464, 15552, 15488, 15616, 15168, 15680, 16000, 15872, 10752, 8576, 8192, 8320, 9728, 8672, 8608, 8384, 11264, 9024, 8992, 12160, 8544, 11520, 11008, 8512, 9008, 12032, 11776, 10240, 8448, 8960, 8640, 9040, 8688, 9048, 15840, 16288, 15856, 16128, 16224, 16368, 40960, 6144, 0, 2048, 24576, 7680, 6656, 3072, 49152, 13312, 12800, 63488, 5632, 53248, 45056, 5120, 13056, 61440, 57344, 32768, 4096, 12288, 7168, 13568, 7936, 13696, 15776, 16320, 15808, 16352}; 30 | unsigned char l_95[95] = { 3, 11, 11, 11, 12, 12, 9, 10, 11, 11, 11, 11, 10, 10, 9, 10, 10, 10, 11, 11, 11, 11, 11, 12, 12, 10, 10, 10, 10, 11, 11, 10, 9, 8, 11, 9, 10, 7, 12, 11, 10, 8, 12, 12, 9, 11, 8, 8, 11, 12, 9, 8, 7, 10, 11, 11, 13, 12, 13, 12, 11, 12, 10, 11, 12, 4, 7, 5, 6, 3, 8, 7, 6, 4, 8, 8, 5, 7, 4, 4, 7, 8, 5, 4, 3, 6, 7, 7, 9, 8, 9, 11, 11, 11, 12}; 31 | //unsigned char c[] = { ' ', '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~'}; 32 | char SET2_STR[] = {'9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '.', ',', '-', '/', '=', '+', ' ', '(', ')', '$', '%', '&', ';', ':', '<', '>', '*', '"', '{', '}', '[', ']', '@', '?', '\'', '^', '#', '_', '!', '\\', '|', '~', '`', '\0'}; 33 | 34 | enum {SHX_STATE_1 = 1, SHX_STATE_2}; 35 | 36 | byte to_match_repeats_earlier = 1; 37 | byte to_match_repeats_within = 1; 38 | #define USE_64K_LOOKUP 0 39 | #if USE_64K_LOOKUP == 1 40 | byte lookup[65536]; 41 | #endif 42 | #define NICE_LEN_FOR_PRIOR 7 43 | #define NICE_LEN_FOR_OTHER 12 44 | 45 | unsigned int mask[] = {0x8000, 0xC000, 0xE000, 0xF000, 0xF800, 0xFC00, 0xFE00, 0xFF00}; 46 | int append_bits(char *out, int ol, unsigned int code, int clen, byte state) { 47 | 48 | byte cur_bit; 49 | byte blen; 50 | unsigned char a_byte; 51 | 52 | if (state == SHX_STATE_2) { 53 | // remove change state prefix 54 | if ((code >> 9) == 0x1C) { 55 | code <<= 7; 56 | clen -= 7; 57 | } 58 | //if (code == 14272 && clen == 10) { 59 | // code = 9084; 60 | // clen = 14; 61 | //} 62 | } 63 | while (clen > 0) { 64 | cur_bit = ol % 8; 65 | blen = (clen > 8 ? 8 : clen); 66 | a_byte = (code & mask[blen - 1]) >> 8; 67 | a_byte >>= cur_bit; 68 | if (blen + cur_bit > 8) 69 | blen = (8 - cur_bit); 70 | if (cur_bit == 0) 71 | out[ol / 8] = a_byte; 72 | else 73 | out[ol / 8] |= a_byte; 74 | code <<= blen; 75 | ol += blen; 76 | clen -= blen; 77 | } 78 | return ol; 79 | } 80 | 81 | int encodeCount(char *out, int ol, int count) { 82 | const byte codes[7] = {0x01, 0x82, 0xC3, 0xE5, 0xED, 0xF5, 0xFD}; 83 | const byte bit_len[7] = {2, 5, 7, 9, 12, 16, 17}; 84 | const uint16_t adder[7] = {0, 4, 36, 164, 676, 4772, 0}; 85 | int till = 0; 86 | for (int i = 0; i < 6; i++) { 87 | till += (1 << bit_len[i]); 88 | if (count < till) { 89 | ol = append_bits(out, ol, (codes[i] & 0xF8) << 8, codes[i] & 0x07, 1); 90 | ol = append_bits(out, ol, (count - adder[i]) << (16 - bit_len[i]), bit_len[i], 1); 91 | return ol; 92 | } 93 | } 94 | return ol; 95 | } 96 | 97 | int matchOccurance(const char *in, int len, int l, char *out, int *ol) { 98 | int j, k; 99 | for (j = 0; j < l; j++) { 100 | for (k = j; k < l && (l + k - j) < len; k++) { 101 | if (in[k] != in[l + k - j]) 102 | break; 103 | } 104 | if ((k - j) > (NICE_LEN_FOR_PRIOR - 1)) { 105 | *ol = append_bits(out, *ol, 14144, 10, 1); 106 | *ol = encodeCount(out, *ol, k - j - NICE_LEN_FOR_PRIOR); // len 107 | *ol = encodeCount(out, *ol, l - j - NICE_LEN_FOR_PRIOR + 1); // dist 108 | l += (k - j); 109 | l--; 110 | return l; 111 | } 112 | } 113 | return -l; 114 | } 115 | 116 | int matchLine(const char *in, int len, int l, char *out, int *ol, struct lnk_lst *prev_lines) { 117 | int last_ol = *ol; 118 | int last_len = 0; 119 | int last_dist = 0; 120 | int last_ctx = 0; 121 | int line_ctr = 0; 122 | do { 123 | int i, j, k; 124 | int line_len = strlen(prev_lines->data); 125 | for (j = 0; j < line_len; j++) { 126 | for (i = l, k = j; k < line_len && i < len; k++, i++) { 127 | if (prev_lines->data[k] != in[i]) 128 | break; 129 | } 130 | if ((k - j) >= NICE_LEN_FOR_OTHER) { 131 | if (last_len) { 132 | if (j > last_dist) 133 | continue; 134 | //int saving = ((k - j) - last_len) + (last_dist - j) + (last_ctx - line_ctr); 135 | //if (saving < 0) { 136 | // //printf("No savng: %d\n", saving); 137 | // continue; 138 | //} 139 | *ol = last_ol; 140 | } 141 | last_len = (k - j); 142 | last_dist = j; 143 | last_ctx = line_ctr; 144 | *ol = append_bits(out, *ol, 14080, 10, 1); 145 | *ol = encodeCount(out, *ol, last_len - NICE_LEN_FOR_OTHER); 146 | *ol = encodeCount(out, *ol, last_dist); 147 | *ol = encodeCount(out, *ol, last_ctx); 148 | /* 149 | if ((*ol - last_ol) > (last_len * 4)) { 150 | last_len = 0; 151 | *ol = last_ol; 152 | }*/ 153 | //printf("Len: %d, Dist: %d, Line: %d\n", last_len, last_dist, last_ctx); 154 | } 155 | } 156 | line_ctr++; 157 | prev_lines = prev_lines->previous; 158 | } while (prev_lines && prev_lines->data != NULL); 159 | if (last_len) { 160 | l += last_len; 161 | l--; 162 | return l; 163 | } 164 | return -l; 165 | } 166 | 167 | int shox96_0_2_compress(const char *in, int len, char *out, struct lnk_lst *prev_lines) { 168 | 169 | char *ptr; 170 | byte bits; 171 | byte state; 172 | 173 | int l, ll, ol; 174 | char c_in, c_next, c_prev; 175 | byte is_upper, is_all_upper; 176 | 177 | ol = 0; 178 | c_prev = 0; 179 | #if USE_64K_LOOKUP == 1 180 | memset(lookup, 0, sizeof(lookup)); 181 | #endif 182 | state = SHX_STATE_1; 183 | is_all_upper = 0; 184 | for (l=0; l 0) { 209 | c_prev = in[l - 1]; 210 | continue; 211 | } 212 | l = -l; 213 | #if USE_64K_LOOKUP == 1 214 | } else 215 | lookup[to_lookup] = 1; 216 | #endif 217 | } 218 | if (l < (len - NICE_LEN_FOR_OTHER) && to_match_repeats_earlier) { 219 | if (prev_lines != NULL) { 220 | l = matchLine(in, len, l, out, &ol, prev_lines); 221 | if (l > 0) { 222 | c_prev = in[l - 1]; 223 | continue; 224 | } 225 | l = -l; 226 | } 227 | } 228 | if (state == SHX_STATE_2) { 229 | if (c_in == ' ' && len - 1 > l) 230 | ptr = (char *) memchr(SET2_STR, in[l+1], 42); 231 | else 232 | ptr = (char *) memchr(SET2_STR, c_in, 42); 233 | if (ptr == NULL) { 234 | state = SHX_STATE_1; 235 | ol = append_bits(out, ol, 8192, 4, 1); 236 | } 237 | } 238 | is_upper = 0; 239 | if (c_in >= 'A' && c_in <= 'Z') 240 | is_upper = 1; 241 | else { 242 | if (is_all_upper) { 243 | is_all_upper = 0; 244 | ol = append_bits(out, ol, 8192, 4, state); 245 | } 246 | } 247 | if (is_upper && !is_all_upper) { 248 | for (ll=l+5; ll>=l && ll= 'a' && in[ll] <= 'z') 250 | break; 251 | } 252 | if (ll == l-1) { 253 | ol = append_bits(out, ol, 8704, 8, state); 254 | is_all_upper = 1; 255 | } 256 | } 257 | if (state == SHX_STATE_1 && c_in >= '0' && c_in <= '9') { 258 | ol = append_bits(out, ol, 14336, 7, state); 259 | state = SHX_STATE_2; 260 | } 261 | c_next = 0; 262 | if (l+1 < len) 263 | c_next = in[l+1]; 264 | 265 | c_prev = c_in; 266 | if (c_in >= 32 && c_in <= 126) { 267 | c_in -= 32; 268 | if (is_all_upper && is_upper) 269 | c_in += 32; 270 | if (c_in == 0 && state == SHX_STATE_2) 271 | ol = append_bits(out, ol, 15232, 11, state); 272 | else 273 | ol = append_bits(out, ol, c_95[c_in], l_95[c_in], state); 274 | } else 275 | if (c_in == 13 && c_next == 10) { 276 | ol = append_bits(out, ol, 13824, 9, state); 277 | l++; 278 | c_prev = 10; 279 | } else 280 | if (c_in == 10) { 281 | ol = append_bits(out, ol, 13952, 9, state); 282 | } else 283 | if (c_in == 13) { 284 | ol = append_bits(out, ol, 9064, 13, state); 285 | } else 286 | if (c_in == '\t') { 287 | ol = append_bits(out, ol, 9216, 7, state); 288 | } 289 | } 290 | bits = ol % 8; 291 | if (bits) { 292 | ol = append_bits(out, ol, 14272, 8 - bits, 1); 293 | } 294 | //printf("\n%ld\n", ol); 295 | return ol/8+(ol%8?1:0); 296 | 297 | } 298 | 299 | // Decoder is designed for using less memory, not speed 300 | // Decode lookup table for code index and length 301 | // First 2 bits 00, Next 3 bits indicate index of code from 0, 302 | // last 3 bits indicate code length in bits 303 | // 0, 1, 2, 3, 4, 304 | char vcode[32] = {2 + (0 << 3), 3 + (3 << 3), 3 + (1 << 3), 4 + (6 << 3), 0, 305 | // 5, 6, 7, 8, 9, 10 306 | 4 + (4 << 3), 3 + (2 << 3), 4 + (8 << 3), 0, 0, 0, 307 | // 11, 12, 13, 14, 15 308 | 4 + (7 << 3), 0, 4 + (5 << 3), 0, 5 + (9 << 3), 309 | // 16, 17, 18, 19, 20, 21, 22, 23 310 | 0, 0, 0, 0, 0, 0, 0, 0, 311 | // 24, 25, 26, 27, 28, 29, 30, 31 312 | 0, 0, 0, 0, 0, 0, 0, 5 + (10 << 3)}; 313 | // 0, 1, 2, 3, 4, 5, 6, 7, 314 | char hcode[32] = {1 + (1 << 3), 2 + (0 << 3), 0, 3 + (2 << 3), 0, 0, 0, 5 + (3 << 3), 315 | // 8, 9, 10, 11, 12, 13, 14, 15, 316 | 0, 0, 0, 0, 0, 0, 0, 5 + (5 << 3), 317 | // 16, 17, 18, 19, 20, 21, 22, 23 318 | 0, 0, 0, 0, 0, 0, 0, 5 + (4 << 3), 319 | // 24, 25, 26, 27, 28, 29, 30, 31 320 | 0, 0, 0, 0, 0, 0, 0, 5 + (6 << 3)}; 321 | 322 | enum {SHX_SET1 = 0, SHX_SET1A, SHX_SET1B, SHX_SET2, SHX_SET3, SHX_SET4, SHX_SET4A}; 323 | char sets[][11] = {{' ', ' ', 'e', 't', 'a', 'o', 'i', 'n', 's', 'r', 'l'}, 324 | {'c', 'd', 'h', 'u', 'p', 'm', 'b', 'g', 'w', 'f', 'y'}, 325 | {'v', 'k', 'q', 'j', 'x', 'z', ' ', ' ', ' ', ' ', ' '}, 326 | {' ', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8'}, 327 | {'.', ',', '-', '/', '=', '+', ' ', '(', ')', '$', '%'}, 328 | {'&', ';', ':', '<', '>', '*', '"', '{', '}', '[', ']'}, 329 | {'@', '?', '\'', '^', '#', '_', '!', '\\', '|', '~', '`'}}; 330 | 331 | int getBitVal(const char *in, int bit_no, int count) { 332 | return (in[bit_no >> 3] & (0x80 >> (bit_no % 8)) ? 1 << count : 0); 333 | } 334 | 335 | int getCodeIdx(char *code_type, const char *in, int len, int *bit_no_p) { 336 | int code = 0; 337 | int count = 0; 338 | do { 339 | if (*bit_no_p >= len) 340 | return 199; 341 | code += getBitVal(in, *bit_no_p, count); 342 | (*bit_no_p)++; 343 | count++; 344 | if (code_type[code] && 345 | (code_type[code] & 0x07) == count) { 346 | return code_type[code] >> 3; 347 | } 348 | } while (count < 5); 349 | return 1; // skip if code not found 350 | } 351 | 352 | int getNumFromBits(const char *in, int bit_no, int count) { 353 | int ret = 0; 354 | while (count--) { 355 | ret += getBitVal(in, bit_no++, count); 356 | } 357 | return ret; 358 | } 359 | 360 | int readCount(const char *in, int *bit_no_p, int len) { 361 | const byte bit_len[7] = {5, 2, 7, 9, 12, 16, 17}; 362 | const uint16_t adder[7] = {4, 0, 36, 164, 676, 4772, 0}; 363 | int idx = getCodeIdx(hcode, in, len, bit_no_p); 364 | if (idx > 6) 365 | return 0; 366 | int count = getNumFromBits(in, *bit_no_p, bit_len[idx]) + adder[idx]; 367 | (*bit_no_p) += bit_len[idx]; 368 | return count; 369 | } 370 | 371 | int shox96_0_2_decompress(const char *in, int len, char *out, struct lnk_lst *prev_lines) { 372 | 373 | int dstate; 374 | int bit_no; 375 | byte is_all_upper; 376 | int ol = 0; 377 | bit_no = 0; 378 | dstate = SHX_SET1; 379 | is_all_upper = 0; 380 | 381 | len <<= 3; 382 | out[ol] = 0; 383 | while (bit_no < len) { 384 | int h, v; 385 | char c; 386 | byte is_upper = is_all_upper; 387 | int orig_bit_no = bit_no; 388 | v = getCodeIdx(vcode, in, len, &bit_no); 389 | if (v == 199) { 390 | bit_no = orig_bit_no; 391 | break; 392 | } 393 | h = dstate; 394 | if (v == 0) { 395 | h = getCodeIdx(hcode, in, len, &bit_no); 396 | if (h == 199) { 397 | bit_no = orig_bit_no; 398 | break; 399 | } 400 | if (h == SHX_SET1) { 401 | if (dstate == SHX_SET1) { 402 | if (is_all_upper) { 403 | is_upper = is_all_upper = 0; 404 | continue; 405 | } 406 | v = getCodeIdx(vcode, in, len, &bit_no); 407 | if (v == 199) { 408 | bit_no = orig_bit_no; 409 | break; 410 | } 411 | if (v == 0) { 412 | h = getCodeIdx(hcode, in, len, &bit_no); 413 | if (h == 199) { 414 | bit_no = orig_bit_no; 415 | break; 416 | } 417 | if (h == SHX_SET1) { 418 | is_all_upper = 1; 419 | continue; 420 | } 421 | } 422 | is_upper = 1; 423 | } else { 424 | dstate = SHX_SET1; 425 | continue; 426 | } 427 | } else 428 | if (h == SHX_SET2) { 429 | if (dstate == SHX_SET1) 430 | dstate = SHX_SET2; 431 | continue; 432 | } 433 | if (h != SHX_SET1) { 434 | v = getCodeIdx(vcode, in, len, &bit_no); 435 | if (v == 199) { 436 | bit_no = orig_bit_no; 437 | break; 438 | } 439 | } 440 | } 441 | if (h < 64 && v < 32) 442 | c = sets[h][v]; 443 | if (c >= 'a' && c <= 'z') { 444 | if (is_upper) 445 | c -= 32; 446 | } else { 447 | if (is_upper && dstate == SHX_SET1 && v == 1) 448 | c = '\t'; 449 | if (h == SHX_SET1B) { 450 | switch (v) { 451 | case 6: 452 | out[ol++] = '\r'; 453 | c = '\n'; 454 | break; 455 | case 7: 456 | c = is_upper ? '\r' : '\n'; 457 | break; 458 | case 8: 459 | if (getBitVal(in, bit_no++, 0)) { 460 | int dict_len = readCount(in, &bit_no, len) + NICE_LEN_FOR_PRIOR; 461 | int dist = readCount(in, &bit_no, len) + NICE_LEN_FOR_PRIOR - 1; 462 | memcpy(out + ol, out + ol - dist, dict_len); 463 | ol += dict_len; 464 | } else { 465 | int dict_len = readCount(in, &bit_no, len) + NICE_LEN_FOR_OTHER; 466 | int dist = readCount(in, &bit_no, len); 467 | int ctx = readCount(in, &bit_no, len); 468 | struct lnk_lst *cur_line = prev_lines; 469 | while (ctx--) 470 | cur_line = cur_line->previous; 471 | memmove(out + ol, cur_line->data + dist, dict_len); 472 | ol += dict_len; 473 | } 474 | continue; 475 | case 9: { 476 | int count = readCount(in, &bit_no, len); 477 | count += 4; 478 | char rpt_c = out[ol - 1]; 479 | while (count--) 480 | out[ol++] = rpt_c; 481 | continue; 482 | } 483 | case 10: 484 | continue; 485 | } 486 | } 487 | } 488 | out[ol++] = c; 489 | } 490 | 491 | return ol; 492 | 493 | } 494 | --------------------------------------------------------------------------------