├── .gitignore ├── CMakeLists.txt ├── ESP32_MSD_Shield_Wiring.jpg ├── ESP_WROOM_32_breakout.png ├── LICENSE ├── README.md ├── console_screenshot.png ├── docs ├── README.md └── _config.yml ├── esp32_sqlite_cover.png ├── examples ├── sqlite3_bulk_data_insert │ └── sqlite3_bulk_data_insert.ino ├── sqlite3_console │ └── sqlite3_console.ino ├── sqlite3_insert_long_blob │ └── sqlite3_insert_long_blob.ino ├── sqlite3_littlefs │ └── sqlite3_littlefs.ino ├── sqlite3_sdmmc │ ├── data │ │ ├── babynames-gendered-2015.db │ │ ├── census2000names.db │ │ └── mdr512.db │ └── sqlite3_sdmmc.ino ├── sqlite3_sdspi │ └── sqlite3_sdspi.ino ├── sqlite3_spiffs │ └── sqlite3_spiffs.ino ├── sqlite3_stackoverflow_users │ └── sqlite3_stackoverflow_users.ino ├── sqlite3_web_console │ └── sqlite3_web_console.ino └── sqlite3_webquery │ ├── data │ └── babyname.db │ └── sqlite3_webquery.ino ├── keywords.txt ├── lib_mgr_ss.png ├── lib_mgr_ss1.png ├── library.properties ├── output_screenshot.png ├── output_shox96.png ├── output_web_1.png ├── output_web_1_2.png ├── output_web_2.png ├── output_web_console.png ├── output_web_so.png ├── output_web_so_id.png ├── output_web_so_loc.png ├── output_web_so_name.png └── src ├── config_ext.h ├── esp32.cpp ├── shox96_0_2.cpp ├── shox96_0_2.h ├── sqlite3.c ├── sqlite3.h ├── unishox1.c └── unishox1.h /.gitignore: -------------------------------------------------------------------------------- 1 | **/.DS_Store 2 | **/.vscode 3 | 4 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(COMPONENT_SRCDIRS 2 | "src" 3 | ) 4 | 5 | set(COMPONENT_ADD_INCLUDEDIRS 6 | "src" 7 | ) 8 | 9 | # Arduino-ESP32 core https://github.com/espressif/arduino-esp32 10 | set(COMPONENT_REQUIRES 11 | "arduino-esp32" 12 | ) 13 | 14 | register_component() 15 | -------------------------------------------------------------------------------- /ESP32_MSD_Shield_Wiring.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/ESP32_MSD_Shield_Wiring.jpg -------------------------------------------------------------------------------- /ESP_WROOM_32_breakout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/ESP_WROOM_32_breakout.png -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sqlite3 Arduino library for ESP32 2 | 3 | Note: This is a general purpose library based on the Sqlite codebase. For logging sensor data into database please use [Sqlite Micro Logger](https://github.com/siara-cc/sqlite_micro_logger_arduino), which is faster and memory efficient. 4 | 5 | This library enables access to SQLite database files from SPIFFS or SD Cards through ESP32 SoC. Given below is a picture of a board that has a ready-made Micro SD slot (using SDMMC 4 bit mode - see example sqlite3_sdmmc): 6 | 7 | ![](ESP_WROOM_32_breakout.png?raw=true) 8 | 9 | Also shown below is the wiring between ESP-WROOM-32 breakout board and Micro SD Shield (using SPI mode - see example sqlite3_sdspi): 10 | 11 | ![](ESP32_MSD_Shield_Wiring.jpg?raw=true) 12 | 13 | ## Why Sqlite on ESP32 is exciting? 14 | 15 | [Sqlite3](http://sqlite.org) is the favourite database of all that is portable and widely used. Availability on ESP32 platform makes it even more portable. Sqlite can handle terrabyte sized data, ACID compliant and guaranteed to be stable. 16 | 17 | So far IoT systems could offer SD Card access, which enabled gigabyte size files accessible, but was not fully utilized because the libraries that were available so far (such as [Arduino Extended Database Library](https://github.com/jwhiddon/EDB) or [SimpleDB](http://www.kendziorra.nl/arduino/103-simpledb-simple-flexible-and-smal)) offered only record number based or linear search and are terribly slow for key based indexed search. 18 | 19 | Sqlite stores data in [B+Tree pages](https://en.wikipedia.org/wiki/B%2B_tree) and so can locate data from millions of records using variable length keys without exerting stress on CPU or RAM. This is demonstrated using the sample databases provided in the example sections. 20 | 21 | Even with the 500 odd kilbytes RAM available on ESP32, millions of records and gigabyte sized databases can be accessed. 22 | 23 | ## Usage 24 | 25 | Sqlite3 C API such as `sqlite3_open` can be directly invoked. Before calling please invoke: 26 | 27 | ```c++ 28 | SD_MMC.begin(); // for Cards attached to the High speed 4-bit port 29 | SPI.begin(); SD.begin(); // for Cards attached to the SPI bus 30 | SPIFFS.begin(); // For SPIFFS 31 | ``` 32 | as appropriate. 33 | 34 | The ESP32 Arduino library has an excellent VFS layer. Even multiple cards can be supported on the SPI bus by specifying the pin number and mount point using the `begin()` method. 35 | 36 | The default mount points are: 37 | ```c++ 38 | '/sdcard' // for SD_MMC 39 | '/sd' // for SD on SPI 40 | '/spiffs' // For SPIFFS 41 | ``` 42 | 43 | and the filenames are to be prefixed with these paths in the `sqlite3_open()` function (such as `sqlite3_open("/spiffs/my.db")`). 44 | 45 | Please see the examples for full illustration of usage for the different file systems. The sample databases given (under `examples/sqlite3_sdmmc/data` folder) need to be copied to the Micro SD card root folder before the SD examples can be used. Please see the comments section of the example. 46 | 47 | ## Wiring 48 | 49 | While there is no wiring needed for SPIFFS, for attaching cards to SPI bus, please use the following connections: 50 | 51 | ```c++ 52 | * SD Card | ESP32 53 | * DAT2 (1) - 54 | * DAT3 (2) SS (D5) 55 | * CMD (3) MOSI (D23) 56 | * VDD (4) 3.3V 57 | * CLK (5) SCK (D19) 58 | * VSS (6) GND 59 | * DAT0 (7) MISO (D18) 60 | * DAT1 (8) - 61 | ``` 62 | 63 | And for SD card attached to High-speed 4-bit SD_MMC port, use: 64 | 65 | ```c++ 66 | * SD Card | ESP32 67 | * DAT2 (1) D12 68 | * DAT3 (2) D13 69 | * CMD (3) D15 70 | * VDD (4) 3.3V 71 | * CLK (5) D14 72 | * VSS (6) GND 73 | * DAT0 (7) D2 74 | * DAT1 (8) D4 75 | ``` 76 | 77 | If you are using a board such as shown in the picture above, this wiring is ready-made. 78 | 79 | ## Installation 80 | 81 | Please download this library, unzip it to the libraries folder of your ESP32 sdk location. The location varies according to your OS. For example, it is usually found in the following locations: 82 | ``` 83 | Windows: C:\Users\(username)\AppData\Roaming\Arduino15 84 | Linux: /home//.arduino15 85 | MacOS: /home//Library/Arduino15 86 | ``` 87 | Under Arduino15 folder please navigate to `packages/esp32/hardware/esp32//libraries` 88 | 89 | If you do not have the ESP32 sdk for Arduino, please see https://github.com/espressif/arduino-esp32 for installing it. 90 | 91 | ## Dependencies / pre-requisites 92 | 93 | No dependencies except for the Arduino and ESP32 core SDK. The Sqlite3 code is included with the library. 94 | 95 | ## Limitations on ESP32 96 | 97 | * No serious limitations, except its a bit slow on large datasets. It takes around 700 ms to retrieve from a dataset containing 10 million rows, even using the index. 98 | 99 | ## Limitations of this library 100 | 101 | * Locking is not implemented. So it cannot be reliably used in a multi-threaded / multi-core code set, except for read-only operations. 102 | 103 | ## Limitations of Flash memory 104 | 105 | Any Flash memory such as those available on SPIFFS or Micro SD cards have limitation on number of writes / erase per sector. Usually the limitation is 10000 writes or 100000 writes (on the same sector). Although ESP32 supports wear-levelling, this is to be kept in mind before venturing into write-intensive database projects. There is no limitation on reading from Flash. 106 | 107 | ## Compression with Shox96 108 | 109 | (Shox96 is deprecated and Unishox explained below will be supported in future). 110 | 111 | This implementation of `sqlite3` includes two functions `shox96_0_2c()` and `shox96_0_2d()` for compressing and decompressing text data. 112 | 113 | Shox96 is a compression technique developed for reducing storage size of Short Strings. Details of how it works can be found [here](https://github.com/siara-cc/Shox96). 114 | 115 | As of now it can work on only strings made of 'A to Z', 'a to z', '0-9', Special Characters such as &*() etc. found on keyboard, CR, LF, TAB and Space. 116 | 117 | In general it can achieve upto 40% size reduction for Short Strings. 118 | 119 | ### Usage 120 | 121 | The following set of commands demonstrate how compression can be accomplished: 122 | 123 | ```sql 124 | create table test (b1 blob); 125 | insert into test values (shox96_0_2c('Hello World')); 126 | insert into test values (shox96_0_2c('Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry''s standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book.')); 127 | select txt, length(txt) txt_len from (select shox96_0_2d(b1) txt from test); 128 | select length(b1) compressed_len from test; 129 | ``` 130 | 131 | See screenshots section for output. 132 | 133 | ### Limitations (for Shox96) 134 | 135 | - Trying to decompress any blob that was not compressed using `shox96_0_2c()` will crash the program. 136 | - It does not work if the string has binary characters. that is, other than ASCII 32 to 126, CR, LF and Tab. 137 | - Dictionary based compression / decompression is not yet implemented. 138 | 139 | ## Compression with Unishox 140 | 141 | This implementation also includes two functions `unishox1c()` and `unishox1d()` for compressing and decompressing text data. 142 | 143 | Unishox is a compression technique developed for reducing storage size of Short Unicode Strings. Details of how it works can be found [here](https://github.com/siara-cc/Unishox). 144 | 145 | In general it can achieve upto 40% size reduction for Short Strings. 146 | 147 | ### Usage 148 | 149 | The usage is similar to that of Shox96, only in this case UTF-8 strings can be used. 150 | 151 | See screenshots section for output. 152 | 153 | ### Limitations (for Unishox) 154 | 155 | - Trying to decompress any blob that was not compressed using `unishox1c()` will crash the program. 156 | 157 | ## Acknowledgements 158 | 159 | * This library was developed based on NodeMCU module developed by [Luiz Felipe Silva](https://github.com/luizfeliperj). The documentation can be found [here](https://nodemcu.readthedocs.io/en/master/en/modules/sqlite3/). 160 | * The census2000 and baby names databases were taken from here: http://2016.padjo.org/tutorials/sqlite-data-starterpacks/. But no license information is available. 161 | * The mdr512.db (Million Domain Rank database) was created with data from [The Majestic Million](https://majestic.com/reports/majestic-million) and is provided under CC 3.0 Attribution license. 162 | * The [ESP32 core for Arduino](https://github.com/espressif/arduino-esp32) 163 | * [The Arduino platform](https://arduino.cc) 164 | 165 | ## Screenshots 166 | 167 | ### Output of Micro SD example 168 | 169 | ![](output_screenshot.png?raw=true) 170 | 171 | ### Output of SD Card database query through WebServer example 172 | 173 | ![](output_web_1.png?raw=true) 174 | ![](output_web_2.png?raw=true) 175 | 176 | ### SQLite console 177 | 178 | ![](console_screenshot.png?raw=true) 179 | 180 | ### Unishox compression 181 | 182 | ![](output_web_console.png?raw=true) 183 | 184 | ### Output of Querying StackOverflow DB through WebServer example: 185 | 186 | ![](output_web_so.png?raw=true) 187 | ![](output_web_so_id.png?raw=true) 188 | ![](output_web_so_name.png?raw=true) 189 | ![](output_web_so_loc.png?raw=true) 190 | 191 | ## Issues 192 | 193 | Please contact the author or create issue here if you face problems. 194 | -------------------------------------------------------------------------------- /console_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/console_screenshot.png -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | Hello world 2 | -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-cayman -------------------------------------------------------------------------------- /esp32_sqlite_cover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/esp32_sqlite_cover.png -------------------------------------------------------------------------------- /examples/sqlite3_bulk_data_insert/sqlite3_bulk_data_insert.ino: -------------------------------------------------------------------------------- 1 | /* 2 | This example demonstrates how SQLite behaves 3 | when memory is low. 4 | It shows how heap defragmentation causes 5 | out of memory and how to avoid it. 6 | At first it asks how much memory to occupy 7 | so as not be available to SQLite. Then 8 | tries to insert huge number of records 9 | and shows how much free memory available 10 | after each insert. 11 | */ 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include "SD_MMC.h" 18 | #include "SPIFFS.h" 19 | 20 | char *dat = NULL; 21 | void block_heap(int times) { 22 | 23 | while (times--) { 24 | dat = (char *) malloc(4096); 25 | } 26 | 27 | } 28 | 29 | const char* data = "Callback function called"; 30 | static int callback(void *data, int argc, char **argv, char **azColName){ 31 | int i; 32 | Serial.printf("%s: ", (const char*)data); 33 | for (i = 0; i= ' ' && str[ctr] <= '~') 75 | ctr++; 76 | if (ctr >= max_len) 77 | break; 78 | } 79 | } 80 | str[ctr] = 0; 81 | Serial.println(str); 82 | } 83 | 84 | int input_num() { 85 | char in[20]; 86 | int ctr = 0; 87 | in[ctr] = 0; 88 | while (in[ctr] != '\n') { 89 | if (Serial.available()) { 90 | in[ctr] = Serial.read(); 91 | if (in[ctr] >= '0' && in[ctr] <= '9') 92 | ctr++; 93 | if (ctr >= sizeof(in)) 94 | break; 95 | } 96 | } 97 | in[ctr] = 0; 98 | int ret = atoi(in); 99 | Serial.println(ret); 100 | return ret; 101 | } 102 | 103 | void displayPrompt(const char *title) { 104 | Serial.print(F("Enter ")); 105 | Serial.println(title); 106 | } 107 | 108 | void displayFreeHeap() { 109 | Serial.printf("\nHeap size: %d\n", ESP.getHeapSize()); 110 | Serial.printf("Free Heap: %d\n", heap_caps_get_free_size(MALLOC_CAP_8BIT)); 111 | Serial.printf("Min Free Heap: %d\n", heap_caps_get_minimum_free_size(MALLOC_CAP_8BIT)); 112 | Serial.printf("Max Alloc Heap: %d\n", heap_caps_get_largest_free_block(MALLOC_CAP_8BIT)); 113 | } 114 | 115 | char *random_strings[] = {"Hello world", "Have a nice day", "Testing memory problems", "This should work", "ESP32 has 512k RAM", "ESP8266 has only 36k user RAM", 116 | "A stitch in time saves nine", "Needle in a haystack", "Too many strings", "I am done"}; 117 | char sql[1024]; 118 | sqlite3 *db1; 119 | sqlite3_stmt *res; 120 | const char *tail; 121 | int rc; 122 | 123 | void setup() { 124 | Serial.begin(115200); 125 | if (!SPIFFS.begin(true)) { 126 | Serial.println(F("Failed to mount file Serial")); 127 | return; 128 | } 129 | 130 | randomSeed(analogRead(0)); 131 | 132 | SPI.begin(); 133 | SD_MMC.begin(); 134 | 135 | displayFreeHeap(); 136 | displayPrompt("No. of 4k heap to block:"); 137 | block_heap(input_num()); 138 | displayFreeHeap(); 139 | 140 | sqlite3_initialize(); 141 | 142 | } 143 | 144 | void loop() { 145 | 146 | // Open database 1 147 | if (openDb("/sdcard/bulk_ins.db", &db1)) 148 | return; 149 | 150 | displayFreeHeap(); 151 | 152 | rc = db_exec(db1, "CREATE TABLE IF NOT EXISTS test (c1 INTEGER, c2, c3, c4, c5 INTEGER, c6 INTEGER, c7, c8, c9 DATETIME, c10 DATETIME, c11 INTEGER )"); 153 | if (rc != SQLITE_OK) { 154 | sqlite3_close(db1); 155 | return; 156 | } 157 | 158 | displayFreeHeap(); 159 | 160 | int rec_count; 161 | displayPrompt("No. of records to insert:"); 162 | rec_count = input_num(); 163 | char *sql = "INSERT INTO test VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; 164 | rc = sqlite3_prepare_v2(db1, sql, strlen(sql), &res, &tail); 165 | if (rc != SQLITE_OK) { 166 | Serial.printf("ERROR preparing sql: %s\n", sqlite3_errmsg(db1)); 167 | sqlite3_close(db1); 168 | return; 169 | } 170 | char *value; 171 | while (rec_count--) { 172 | sqlite3_bind_int(res, 1, random(65535)); 173 | value = random_strings[random(10)]; 174 | sqlite3_bind_text(res, 2, value, strlen(value), SQLITE_STATIC); 175 | value = random_strings[random(10)]; 176 | sqlite3_bind_text(res, 3, value, strlen(value), SQLITE_STATIC); 177 | value = random_strings[random(10)]; 178 | sqlite3_bind_text(res, 4, value, strlen(value), SQLITE_STATIC); 179 | sqlite3_bind_int(res, 5, random(65535)); 180 | sqlite3_bind_int(res, 6, random(65535)); 181 | value = random_strings[random(10)]; 182 | sqlite3_bind_text(res, 7, value, strlen(value), SQLITE_STATIC); 183 | value = random_strings[random(10)]; 184 | sqlite3_bind_text(res, 8, value, strlen(value), SQLITE_STATIC); 185 | sqlite3_bind_int(res, 9, random(100000000L)); 186 | sqlite3_bind_int(res, 10, random(100000000L)); 187 | sqlite3_bind_int(res, 11, random(65535)); 188 | if (sqlite3_step(res) != SQLITE_DONE) { 189 | Serial.printf("ERROR executing stmt: %s\n", sqlite3_errmsg(db1)); 190 | sqlite3_close(db1); 191 | return; 192 | } 193 | sqlite3_clear_bindings(res); 194 | rc = sqlite3_reset(res); 195 | if (rc != SQLITE_OK) { 196 | sqlite3_close(db1); 197 | return; 198 | } 199 | displayFreeHeap(); 200 | } 201 | sqlite3_finalize(res); 202 | Serial.write("\n"); 203 | 204 | rc = db_exec(db1, "Select count(*) from test"); 205 | if (rc != SQLITE_OK) { 206 | sqlite3_close(db1); 207 | return; 208 | } 209 | 210 | sqlite3_close(db1); 211 | displayFreeHeap(); 212 | 213 | displayPrompt("Press enter to continue:"); 214 | input_num(); 215 | 216 | } 217 | -------------------------------------------------------------------------------- /examples/sqlite3_console/sqlite3_console.ino: -------------------------------------------------------------------------------- 1 | /* 2 | Open and execute SQL statements throught this console. 3 | Output is in tab-delimited format. 4 | Connections for SD Card in SPI Mode: 5 | * SD Card | ESP32 6 | * DAT2 - 7 | * DAT3 SS (D5) 8 | * CMD MOSI (D23) 9 | * VSS GND 10 | * VDD 3.3V 11 | * CLK SCK (D18) 12 | * DAT0 MISO (D19) 13 | * DAT1 - 14 | Connections for SD Card in SD_MMC Mode: 15 | * SD Card | ESP32 16 | * DAT2 (1) D12 17 | * DAT3 (2) D13 18 | * CMD (3) D15 19 | * VDD (4) 3.3V 20 | * CLK (5) D14 21 | * VSS (6) GND 22 | * DAT0 (7) D2 23 | * DAT1 (8) D4 24 | */ 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include "SPIFFS.h" 33 | #include "SD_MMC.h" 34 | #include "SD.h" 35 | 36 | /* You only need to format SPIFFS the first time you run a 37 | test or else use the SPIFFS plugin to create a partition 38 | https://github.com/me-no-dev/arduino-esp32fs-plugin */ 39 | #define FORMAT_SPIFFS_IF_FAILED true 40 | #define MAX_FILE_NAME_LEN 100 41 | #define MAX_STR_LEN 500 42 | 43 | char db_file_name[MAX_FILE_NAME_LEN] = "\0"; 44 | sqlite3 *db = NULL; 45 | int rc; 46 | 47 | bool first_time = false; 48 | static int callback(void *data, int argc, char **argv, char **azColName) { 49 | int i; 50 | if (first_time) { 51 | Serial.println((const char *) data); 52 | for (i = 0; i= ' ' && str[ctr] <= '~') 116 | ctr++; 117 | if (ctr >= max_len) 118 | break; 119 | } 120 | } 121 | str[ctr] = 0; 122 | Serial.println(str); 123 | return ctr; 124 | } 125 | 126 | int input_num() { 127 | char in[20]; 128 | int ctr = 0; 129 | in[ctr] = 0; 130 | while (in[ctr] != '\n') { 131 | if (Serial.available()) { 132 | in[ctr] = Serial.read(); 133 | if (in[ctr] >= '0' && in[ctr] <= '9') 134 | ctr++; 135 | if (ctr >= sizeof(in)) 136 | break; 137 | } 138 | } 139 | in[ctr] = 0; 140 | int ret = atoi(in); 141 | Serial.println(ret); 142 | return ret; 143 | } 144 | 145 | void listDir(fs::FS &fs, const char * dirname) { 146 | Serial.print(F("Listing directory: ")); 147 | Serial.println(dirname); 148 | File root = fs.open(dirname); 149 | if (!root){ 150 | Serial.println(F("Failed to open directory")); 151 | return; 152 | } 153 | if (!root.isDirectory()){ 154 | Serial.println("Not a directory"); 155 | return; 156 | } 157 | File file = root.openNextFile(); 158 | while (file) { 159 | if (file.isDirectory()) { 160 | Serial.print(" Dir : "); 161 | Serial.println(file.name()); 162 | } else { 163 | Serial.print(" File: "); 164 | Serial.print(file.name()); 165 | Serial.print(" Size: "); 166 | Serial.println(file.size()); 167 | } 168 | file = root.openNextFile(); 169 | } 170 | } 171 | 172 | void renameFile(fs::FS &fs, const char *path1, const char *path2) { 173 | Serial.printf("Renaming file %s to %s\n", path1, path2); 174 | if (fs.rename(path1, path2)) { 175 | Serial.println(F("File renamed")); 176 | } else { 177 | Serial.println(F("Rename failed")); 178 | } 179 | } 180 | 181 | void deleteFile(fs::FS &fs, const char *path) { 182 | Serial.printf("Deleting file: %s\n", path); 183 | if (fs.remove(path)) { 184 | Serial.println(F("File deleted")); 185 | } else { 186 | Serial.println(F("Delete failed")); 187 | } 188 | } 189 | 190 | enum {CHOICE_OPEN_DB = 1, CHOICE_EXEC_SQL, CHOICE_EXEC_MULTI_SQL, CHOICE_CLOSE_DB, 191 | CHOICE_LIST_FOLDER, CHOICE_RENAME_FILE, CHOICE_DELETE_FILE, CHOICE_SHOW_FREE_MEM}; 192 | int askChoice() { 193 | Serial.println(); 194 | Serial.println(F("Welcome to SQLite console!!")); 195 | Serial.println(F("---------------------------")); 196 | Serial.println(); 197 | Serial.print(F("Database file: ")); 198 | Serial.println(db_file_name); 199 | Serial.println(); 200 | Serial.println(F("1. Open database")); 201 | Serial.println(F("2. Execute SQL")); 202 | Serial.println(F("3. Execute Multiple SQL")); 203 | Serial.println(F("4. Close database")); 204 | Serial.println(F("5. List folder contents")); 205 | Serial.println(F("6. Rename file")); 206 | Serial.println(F("7. Delete file")); 207 | Serial.println(F("8. Show free memory")); 208 | Serial.println(); 209 | Serial.print(F("Enter choice: ")); 210 | return input_num(); 211 | } 212 | 213 | void displayPrompt(const char *title) { 214 | Serial.println(F("(prefix /spiffs/ or /sd/ or /sdcard/ for")); 215 | Serial.println(F(" SPIFFS or SD_SPI or SD_MMC respectively)")); 216 | Serial.print(F("Enter ")); 217 | Serial.println(title); 218 | } 219 | 220 | const char *prefixSPIFFS = "/spiffs/"; 221 | const char *prefixSD_SPI = "/sd/"; 222 | const char *prefixSD_MMC = "/sdcard/"; 223 | fs::FS *ascertainFS(const char *str, int *prefix_len) { 224 | if (strstr(str, prefixSPIFFS) == str) { 225 | *prefix_len = strlen(prefixSPIFFS) - 1; 226 | return &SPIFFS; 227 | } 228 | if (strstr(str, prefixSD_SPI) == str) { 229 | *prefix_len = strlen(prefixSD_SPI) - 1; 230 | return &SD; 231 | } 232 | if (strstr(str, prefixSD_MMC) == str) { 233 | *prefix_len = strlen(prefixSD_MMC) - 1; 234 | return &SD_MMC; 235 | } 236 | return NULL; 237 | } 238 | 239 | void setup() { 240 | Serial.begin(115200); 241 | if (!SPIFFS.begin(FORMAT_SPIFFS_IF_FAILED)) { 242 | Serial.println(F("Failed to mount file Serial")); 243 | return; 244 | } 245 | SPI.begin(); 246 | SD_MMC.begin(); 247 | SD.begin(); 248 | sqlite3_initialize(); 249 | } 250 | 251 | char str[MAX_STR_LEN]; 252 | void loop() { 253 | 254 | int choice = askChoice(); 255 | switch (choice) { 256 | case CHOICE_OPEN_DB: 257 | displayPrompt("file name: "); 258 | input_string(str, MAX_FILE_NAME_LEN); 259 | if (str[0] != 0) { 260 | strncpy(db_file_name, str, MAX_FILE_NAME_LEN); 261 | db_open(); 262 | } 263 | break; 264 | case CHOICE_EXEC_SQL: 265 | Serial.print(F("Enter SQL (max ")); 266 | Serial.print(MAX_STR_LEN); 267 | Serial.println(F(" characters):")); 268 | input_string(str, MAX_STR_LEN); 269 | if (str[0] != 0) 270 | db_exec(str); 271 | break; 272 | case CHOICE_EXEC_MULTI_SQL: 273 | Serial.println(F("(Copy paste may not always work due to limited serial buffer)")); 274 | Serial.println(F("Keep entering SQL, empty to stop:")); 275 | do { 276 | input_string(str, MAX_STR_LEN); 277 | if (str[0] != 0) 278 | db_exec(str); 279 | } while (str[0] != 0); 280 | break; 281 | case CHOICE_CLOSE_DB: 282 | if (db_file_name[0] != 0) { 283 | db_file_name[0] = 0; 284 | sqlite3_close(db); 285 | } 286 | break; 287 | case CHOICE_LIST_FOLDER: 288 | case CHOICE_RENAME_FILE: 289 | case CHOICE_DELETE_FILE: 290 | fs::FS *fs; 291 | displayPrompt("path: "); 292 | input_string(str, MAX_STR_LEN); 293 | if (str[0] != 0) { 294 | int fs_prefix_len = 0; 295 | fs = ascertainFS(str, &fs_prefix_len); 296 | if (fs != NULL) { 297 | switch (choice) { 298 | case CHOICE_LIST_FOLDER: 299 | listDir(*fs, str + fs_prefix_len); 300 | break; 301 | case CHOICE_RENAME_FILE: 302 | char str1[MAX_FILE_NAME_LEN]; 303 | displayPrompt("path to rename as: "); 304 | input_string(str1, MAX_STR_LEN); 305 | if (str1[0] != 0) 306 | renameFile(*fs, str + fs_prefix_len, str1 + fs_prefix_len); 307 | break; 308 | case CHOICE_DELETE_FILE: 309 | deleteFile(*fs, str + fs_prefix_len); 310 | break; 311 | } 312 | } 313 | } 314 | break; 315 | case CHOICE_SHOW_FREE_MEM: 316 | Serial.printf("\nHeap size: %d\n", ESP.getHeapSize()); 317 | Serial.printf("Free Heap: %d\n", esp_get_free_heap_size()); 318 | Serial.printf("Min Free Heap: %d\n", esp_get_minimum_free_heap_size()); 319 | Serial.printf("Largest Free block: %d\n", heap_caps_get_largest_free_block(MALLOC_CAP_8BIT)); 320 | break; 321 | default: 322 | Serial.println(F("Invalid choice. Try again.")); 323 | } 324 | 325 | } -------------------------------------------------------------------------------- /examples/sqlite3_insert_long_blob/sqlite3_insert_long_blob.ino: -------------------------------------------------------------------------------- 1 | /* Example to create a table and insert 5000 bytes into blob column. 2 | Creates the database on SD Card connected to the SDMMC port. 3 | */ 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "SD_MMC.h" 10 | 11 | const char* data = "Callback function called"; 12 | static int callback(void *data, int argc, char **argv, char **azColName){ 13 | int i; 14 | Serial.printf("%s: ", (const char*)data); 15 | for (i = 0; i 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "LittleFS.h" 11 | 12 | /* You only need to format LITTLEFS the first time you run a 13 | test or else use the LITTLEFS plugin to create a partition 14 | https://github.com/lorol/arduino-esp32fs-plugin/releases */ 15 | #define FORMAT_LITTLEFS_IF_FAILED true 16 | 17 | const char* data = "Callback function called"; 18 | static int callback(void *data, int argc, char **argv, char **azColName) { 19 | int i; 20 | Serial.printf("%s: ", (const char*)data); 21 | for (i = 0; i 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "SD_MMC.h" 14 | 15 | const char* data = "Callback function called"; 16 | static int callback(void *data, int argc, char **argv, char **azColName){ 17 | int i; 18 | Serial.printf("%s: ", (const char*)data); 19 | for (i = 0; i 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "SD.h" 24 | 25 | const char* data = "Callback function called"; 26 | static int callback(void *data, int argc, char **argv, char **azColName){ 27 | int i; 28 | Serial.printf("%s: ", (const char*)data); 29 | for (i = 0; i 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "SPIFFS.h" 11 | 12 | /* You only need to format SPIFFS the first time you run a 13 | test or else use the SPIFFS plugin to create a partition 14 | https://github.com/me-no-dev/arduino-esp32fs-plugin */ 15 | #define FORMAT_SPIFFS_IF_FAILED true 16 | 17 | const char* data = "Callback function called"; 18 | static int callback(void *data, int argc, char **argv, char **azColName) { 19 | int i; 20 | Serial.printf("%s: ", (const char*)data); 21 | for (i = 0; i 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include "SD_MMC.h" 54 | 55 | const char *ssid = "Nokia1"; 56 | const char *password = "nokiafour"; 57 | 58 | WebServer server(80); 59 | 60 | const int led = 13; 61 | 62 | void handleRoot() { 63 | digitalWrite ( led, 1 ); 64 | String temp; 65 | int sec = millis() / 1000; 66 | int min = sec / 60; 67 | int hr = min / 60; 68 | 69 | temp = F("\ 70 | ESP32 Demo to Query database on Micro SD\ 71 | \ 74 | \ 75 | \ 76 |

Hello from ESP32!

\ 77 |

Query StackOverflow Users database on Micro SD Card

\ 78 |

StackOverflow publishes snapshot of its data periodically at archive.org \ 79 | here \ 80 | and is \ 81 | licensed under cc-by-sa 3.0.

\ 82 |

This repository \ 83 | hosts StackOverflow User data imported into a convenient SQLite database\ 84 | (size 1.94 GB and contains close to 10 million records). The date of snapshot is 3-Dec-2018.

\ 85 |

This example shows how to retrieve data from this database copied to Micro SD Card \ 86 | attached to ESP32 through its Web Server and display in the form of HTML page.

\ 87 |

Query by User Id

\ 88 |
\ 89 | Enter id: \ 90 | \ 91 |

(To find your id, see search box after clicking your profile icon on https://stackoverflow.com)

\ 92 | \ 93 |
\ 94 |
\ 95 |

Query by Display name

\ 96 |
\ 97 | \ 98 | Enter Display name: \ 99 |

\ 100 |
\ 101 |
\ 102 |

Aggregate Query by Location

\ 103 |
\ 104 | \ 105 | \ 106 | Enter Location (blank for all): \ 107 | \ 108 |
Minimum count:\ 109 | \ 110 |

\ 111 |
\ 112 |
\ 113 | \ 114 | "); 115 | server.send(200, "text/html", temp.c_str()); 116 | digitalWrite(led, 0); 117 | } 118 | 119 | void handleNotFound() { 120 | digitalWrite(led, 1); 121 | String message = F("File Not Found\n\n"); 122 | message += F("URI: "); 123 | message += server.uri(); 124 | message += F("\nMethod: "); 125 | message += (server.method() == HTTP_GET) ? "GET" : "POST"; 126 | message += F("\nArguments: "); 127 | message += server.args(); 128 | message += F("\n"); 129 | for ( uint8_t i = 0; i < server.args(); i++ ) { 130 | message += " " + server.argName(i) + ": " + server.arg(i) + "\n"; 131 | } 132 | server.send(404, "text/plain", message); 133 | digitalWrite(led, 0); 134 | } 135 | 136 | sqlite3 *db1; 137 | int rc; 138 | sqlite3_stmt *res; 139 | int rec_count = 0; 140 | const char *tail; 141 | 142 | int openDb(const char *filename, sqlite3 **db) { 143 | int rc = sqlite3_open(filename, db); 144 | if (rc) { 145 | Serial.print(F("Can't open database: ")); 146 | Serial.println(sqlite3_errmsg(*db)); 147 | return rc; 148 | } else { 149 | Serial.println(F("Opened database successfully")); 150 | } 151 | return rc; 152 | } 153 | 154 | void setup(void) { 155 | pinMode(led, OUTPUT); 156 | digitalWrite(led, 0); 157 | Serial.begin(115200); 158 | WiFi.mode(WIFI_STA); 159 | WiFi.begin(ssid, password); 160 | Serial.println(F("Hello")); 161 | 162 | // Wait for connection 163 | while (WiFi.status() != WL_CONNECTED) { 164 | delay(500); 165 | Serial.print(F(".")); 166 | } 167 | 168 | Serial.println(""); 169 | Serial.print(F("Connected to ")); 170 | Serial.println(ssid); 171 | Serial.print(F("IP address: ")); 172 | Serial.println(WiFi.localIP()); 173 | 174 | if (MDNS.begin("esp32")) { 175 | Serial.println(F("MDNS responder started")); 176 | } 177 | 178 | SD_MMC.begin(); 179 | sqlite3_initialize(); 180 | 181 | // Open database 182 | if (openDb("/sdcard/so_users.db", &db1)) 183 | return; 184 | 185 | server.on("/", handleRoot); 186 | server.on("/query_db", []() { 187 | long start = micros(); 188 | String sql = F("Select Count(*) From SO_Users Where "); 189 | if (server.arg("so_disp_name").length() > 0) { 190 | sql += F("DisplayName = '"); 191 | sql += server.arg("so_disp_name"); 192 | sql += "'"; 193 | } else if (server.arg("so_id").length() > 0) { 194 | sql += F("Id = '"); 195 | sql += server.arg("so_id"); 196 | sql += F("'"); 197 | } else { 198 | sql = ""; 199 | } 200 | int step_res; 201 | if (sql.length() > 0) { 202 | rc = sqlite3_prepare_v2(db1, sql.c_str(), -1, &res, &tail); 203 | if (rc != SQLITE_OK) { 204 | String resp = F("Failed to fetch data: "); 205 | resp += sqlite3_errmsg(db1); 206 | resp += F(".

"); 207 | server.send ( 200, "text/html", resp.c_str()); 208 | Serial.println(resp.c_str()); 209 | return; 210 | } 211 | do { 212 | step_res = sqlite3_step(res); 213 | if (step_res == SQLITE_ROW) { 214 | rec_count = sqlite3_column_int(res, 0); 215 | if (rec_count > 5000) { 216 | String resp = F("Too many records: "); 217 | resp += rec_count; 218 | resp += F(". Please select different range"); 219 | resp += F(".

"); 220 | server.send ( 200, "text/html", resp.c_str()); 221 | Serial.println(resp.c_str()); 222 | sqlite3_finalize(res); 223 | return; 224 | } 225 | } 226 | } while (step_res != SQLITE_DONE && step_res != SQLITE_ERROR); 227 | sqlite3_finalize(res); 228 | } else 229 | rec_count = -1; 230 | 231 | sql = F("Select * From SO_Users Where "); 232 | if (server.arg("so_disp_name").length() > 0) { 233 | sql += F("DisplayName = '"); 234 | sql += server.arg("so_disp_name"); 235 | sql += F("'"); 236 | } else if (server.arg("so_id").length() > 0) { 237 | sql += F("Id = '"); 238 | sql += server.arg("so_id"); 239 | sql += F("'"); 240 | } else { 241 | sql = F("Select Location, Count(*) Count From SO_Users "); 242 | if (server.arg("so_loc").length() > 0) { 243 | sql += F("Where Location = '"); 244 | sql += server.arg("so_loc"); 245 | sql += F("' "); 246 | } else 247 | sql += F("Where Location > '' "); 248 | sql += F("Group by Location "); 249 | if (server.arg("so_loc_count").length() > 0) { 250 | sql += F("Having Count(*) >= "); 251 | sql += server.arg("so_loc_count"); 252 | } 253 | } 254 | rc = sqlite3_prepare_v2(db1, sql.c_str(), -1, &res, &tail); 255 | if (rc != SQLITE_OK) { 256 | String resp = F("Failed to fetch data: "); 257 | resp += sqlite3_errmsg(db1); 258 | resp += F("

back"); 259 | server.send ( 200, "text/html", resp.c_str()); 260 | Serial.println(resp.c_str()); 261 | return; 262 | } 263 | 264 | server.setContentLength(CONTENT_LENGTH_UNKNOWN); 265 | String resp = F("\ 266 | StackOverflow Database query on ESP32 through web server\ 267 |

Query StackOverflow Users db on Micro SD card attached to ESP32 through its web server

"); 270 | resp += sql; 271 | resp += F("

"); 272 | if (rec_count >= 0) { 273 | resp += F("

No. of records: "); 274 | resp += rec_count; 275 | resp += F("

"); 276 | } 277 | resp += F(""); 278 | server.send(200, "text/html", resp.c_str()); 279 | int cols = sqlite3_column_count(res); 280 | resp = F(""); 281 | for (int i = 0; i < cols; i++) { 282 | resp += F(""); 285 | } 286 | resp += F(""); 287 | server.sendContent(resp); 288 | do { 289 | step_res = sqlite3_step(res); 290 | if (step_res == SQLITE_ROW) { 291 | resp = F(""); 292 | for (int i = 0; i < cols; i++) { 293 | resp += F(""); 296 | } 297 | resp += F(""); 298 | server.sendContent(resp); 299 | rec_count++; 300 | } 301 | } while (step_res != SQLITE_DONE && step_res != SQLITE_ERROR); 302 | resp = F("
"); 283 | resp += (const char *) sqlite3_column_name(res, i); 284 | resp += F("
"); 294 | resp += (const char *) sqlite3_column_text(res, i); 295 | resp += F("
"); 303 | resp += F("
Time taken (seconds): "); 304 | resp += (micros()-start)/1000000; 305 | resp += F("

"); 306 | server.sendContent(resp); 307 | sqlite3_finalize(res); 308 | }); 309 | server.onNotFound(handleNotFound); 310 | server.begin(); 311 | Serial.println(F("HTTP server started")); 312 | } 313 | 314 | void loop ( void ) { 315 | server.handleClient(); 316 | } 317 | -------------------------------------------------------------------------------- /examples/sqlite3_web_console/sqlite3_web_console.ino: -------------------------------------------------------------------------------- 1 | /* 2 | This utility provides ability to run Sql Statements on Sqlite3 databases from SD Card 3 | or SPIFFS through the Web Server and display in the form of HTML page. 4 | 5 | For more information, visit https://github.com/siara-cc/esp32_arduino_sqlite3_lib 6 | 7 | Copyright (c) 2018, Siara Logics (cc) 8 | */ 9 | 10 | /* 11 | Copyright (c) 2015, Majenko Technologies 12 | All rights reserved. 13 | 14 | Redistribution and use in source and binary forms, with or without modification, 15 | are permitted provided that the following conditions are met: 16 | 17 | * * Redistributions of source code must retain the above copyright notice, this 18 | list of conditions and the following disclaimer. 19 | 20 | * * Redistributions in binary form must reproduce the above copyright notice, this 21 | list of conditions and the following disclaimer in the documentation and/or 22 | other materials provided with the distribution. 23 | 24 | * * Neither the name of Majenko Technologies nor the names of its 25 | contributors may be used to endorse or promote products derived from 26 | this software without specific prior written permission. 27 | 28 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 29 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 30 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 31 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 32 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 33 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 34 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 35 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 37 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38 | */ 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include "SPIFFS.h" 48 | #include "SD_MMC.h" 49 | #include "SD.h" 50 | 51 | const char *ssid = "Nokia1"; 52 | const char *password = "nokiafive"; 53 | 54 | WebServer server(80); 55 | 56 | const int led = 13; 57 | 58 | void handleRoot(const char *db_name, const char *sql) { 59 | digitalWrite ( led, 1 ); 60 | String temp; 61 | int sec = millis() / 1000; 62 | int min = sec / 60; 63 | int hr = min / 60; 64 | 65 | temp = "\ 66 | ESP32 Sqlite Web Console\ 67 | \ 70 | \ 71 | \ 72 |

ESP32 Sqlite Web Console

\ 73 |

Uptime: "; 74 | temp += hr; 75 | temp += ":"; 76 | temp += min % 60; 77 | temp += ":"; 78 | temp += sec % 60; 79 | temp += "

\ 80 |
\ 81 | \ 85 |
File name (prefix with /spiffs/ or /sd/ or /sdcard/):
\ 89 |

\ 90 |
"; 91 | 92 | server.send ( 200, "text/html", temp.c_str() ); 93 | digitalWrite ( led, 0 ); 94 | } 95 | 96 | void handleNotFound() { 97 | digitalWrite ( led, 1 ); 98 | String message = "File Not Found\n\n"; 99 | message += "URI: "; 100 | message += server.uri(); 101 | message += "\nMethod: "; 102 | message += ( server.method() == HTTP_GET ) ? "GET" : "POST"; 103 | message += "\nArguments: "; 104 | message += server.args(); 105 | message += "\n"; 106 | 107 | for ( uint8_t i = 0; i < server.args(); i++ ) { 108 | message += " " + server.argName ( i ) + ": " + server.arg ( i ) + "\n"; 109 | } 110 | 111 | server.send ( 404, "text/plain", message ); 112 | digitalWrite ( led, 0 ); 113 | } 114 | 115 | sqlite3 *db1; 116 | int rc; 117 | sqlite3_stmt *res; 118 | int rec_count = 0; 119 | const char *tail; 120 | char current_db[255]; 121 | 122 | int openDb(const char *filename) { 123 | if (strncmp(filename, current_db, sizeof(current_db)) == 0) 124 | return 0; 125 | else 126 | sqlite3_close(db1); 127 | int rc = sqlite3_open(filename, &db1); 128 | if (rc) { 129 | Serial.printf("Can't open database: %s\n", sqlite3_errmsg(db1)); 130 | memset(current_db, '\0', sizeof(current_db)); 131 | return rc; 132 | } else { 133 | Serial.printf("Opened database successfully\n"); 134 | strcpy(current_db, filename); 135 | } 136 | return rc; 137 | } 138 | 139 | void setup ( void ) { 140 | pinMode(led, OUTPUT); 141 | digitalWrite(led, 0); 142 | Serial.begin(115200); 143 | WiFi.mode(WIFI_STA); 144 | WiFi.begin(ssid, password); 145 | Serial.println(""); 146 | 147 | // Wait for connection 148 | while ( WiFi.status() != WL_CONNECTED ) { 149 | delay ( 500 ); 150 | Serial.print ( "." ); 151 | } 152 | 153 | Serial.println ( "" ); 154 | Serial.print ( "Connected to " ); 155 | Serial.println ( ssid ); 156 | Serial.print ( "IP address: " ); 157 | Serial.println ( WiFi.localIP() ); 158 | 159 | if ( MDNS.begin ( "esp32" ) ) { 160 | Serial.println ( "MDNS responder started" ); 161 | } 162 | 163 | memset(current_db, '\0', sizeof(current_db)); 164 | 165 | if (!SPIFFS.begin(true)) { 166 | Serial.println(F("Failed to mount file Serial")); 167 | return; 168 | } 169 | SPI.begin(); 170 | SD_MMC.begin(); 171 | SD.begin(); 172 | sqlite3_initialize(); 173 | 174 | server.on ( "/", []() { 175 | handleRoot(NULL, NULL); 176 | }); 177 | server.on ( "/exec_sql", []() { 178 | String db_name = server.arg("db_name"); 179 | String sql = server.arg("sql"); 180 | server.setContentLength(CONTENT_LENGTH_UNKNOWN); 181 | handleRoot(db_name.c_str(), sql.c_str()); 182 | if (openDb(db_name.c_str())) { 183 | String resp = "Error opening database: "; 184 | resp += sqlite3_errmsg(db1); 185 | resp += "
";
186 |         resp += server.arg("db_name");
187 |         resp += "
"; 188 | resp += ".

"; 189 | server.sendContent(resp); 190 | return; 191 | } 192 | //String sql = "Select count(*) from gendered_names where name between '"; 193 | //sql += server.arg("from"); 194 | //sql += "' and '"; 195 | //sql += server.arg("to"); 196 | //sql += "'"; 197 | //rc = sqlite3_prepare_v2(db1, sql.c_str(), -1, &res, &tail); 198 | //if (rc != SQLITE_OK) { 199 | // String resp = "Failed to fetch data: "; 200 | // resp += sqlite3_errmsg(db1); 201 | // resp += ".

"; 202 | // server.send ( 200, "text/html", resp.c_str()); 203 | // Serial.println(resp.c_str()); 204 | // return; 205 | //} 206 | //while (sqlite3_step(res) == SQLITE_ROW) { 207 | // rec_count = sqlite3_column_int(res, 0); 208 | // if (rec_count > 5000) { 209 | // String resp = "Too many records: "; 210 | // resp += rec_count; 211 | // resp += ". Please select different range"; 212 | // resp += ".

"; 213 | // server.send ( 200, "text/html", resp.c_str()); 214 | // Serial.println(resp.c_str()); 215 | // sqlite3_finalize(res); 216 | // return; 217 | // } 218 | //} 219 | //sqlite3_finalize(res); 220 | 221 | rc = sqlite3_prepare_v2(db1, sql.c_str(), -1, &res, &tail); 222 | if (rc != SQLITE_OK) { 223 | String resp = "Failed to fetch data: "; 224 | resp += sqlite3_errmsg(db1); 225 | resp += "

back"; 226 | server.sendContent(resp); 227 | Serial.println(resp.c_str()); 228 | return; 229 | } 230 | 231 | rec_count = 0; 232 | String resp = "

Result:

"; 233 | resp += sql; 234 | resp += "

"; 235 | server.sendContent(resp); 236 | bool first = true; 237 | while (sqlite3_step(res) == SQLITE_ROW) { 238 | resp = ""; 239 | if (first) { 240 | int count = sqlite3_column_count(res); 241 | if (count == 0) { 242 | resp += ""; 243 | rec_count = sqlite3_changes(db1); 244 | break; 245 | } 246 | resp += ""; 247 | for (int i = 0; i"; 269 | server.sendContent(resp); 270 | sqlite3_finalize(res); 271 | } ); 272 | server.onNotFound ( handleNotFound ); 273 | server.begin(); 274 | Serial.println ( "HTTP server started" ); 275 | } 276 | 277 | void loop ( void ) { 278 | server.handleClient(); 279 | } 280 | -------------------------------------------------------------------------------- /examples/sqlite3_webquery/data/babyname.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/examples/sqlite3_webquery/data/babyname.db -------------------------------------------------------------------------------- /examples/sqlite3_webquery/sqlite3_webquery.ino: -------------------------------------------------------------------------------- 1 | /* 2 | This example shows how to retrieve data from Sqlite3 databases from SD Card 3 | through the Web Server and display in the form of HTML page. 4 | It also demonstrates query filtering by parameter passing and chunked encoding. 5 | Before running please copy following files to SD Card: 6 | 7 | data/babyname.db 8 | 9 | This database contains around 30000 baby names and corresponding data. 10 | 11 | For more information, visit https://github.com/siara-cc/esp32_arduino_sqlite3_lib 12 | 13 | Copyright (c) 2018, Siara Logics (cc) 14 | */ 15 | 16 | /* 17 | Copyright (c) 2015, Majenko Technologies 18 | All rights reserved. 19 | 20 | Redistribution and use in source and binary forms, with or without modification, 21 | are permitted provided that the following conditions are met: 22 | 23 | * * Redistributions of source code must retain the above copyright notice, this 24 | list of conditions and the following disclaimer. 25 | 26 | * * Redistributions in binary form must reproduce the above copyright notice, this 27 | list of conditions and the following disclaimer in the documentation and/or 28 | other materials provided with the distribution. 29 | 30 | * * Neither the name of Majenko Technologies nor the names of its 31 | contributors may be used to endorse or promote products derived from 32 | this software without specific prior written permission. 33 | 34 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 35 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 36 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 37 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 38 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 39 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 40 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 41 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 42 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 43 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 44 | */ 45 | 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include "SD_MMC.h" 53 | 54 | const char *ssid = "Nokia1"; 55 | const char *password = "nokiafour"; 56 | 57 | WebServer server(80); 58 | 59 | const int led = 13; 60 | 61 | void handleRoot() { 62 | digitalWrite ( led, 1 ); 63 | String temp; 64 | int sec = millis() / 1000; 65 | int min = sec / 60; 66 | int hr = min / 60; 67 | 68 | temp = "\ 69 | ESP32 Demo\ 70 | \ 73 | \ 74 | \ 75 |

Hello from ESP32!

\ 76 |

Uptime: "; 77 | temp += hr; 78 | temp += ":"; 79 | temp += min % 60; 80 | temp += ":"; 81 | temp += sec % 60; 82 | temp += "

\ 83 |

Query gendered names database

\ 84 |
\ 85 | Enter from: \ 86 |
to: \ 87 |

\ 88 | \ 89 | \ 90 | "; 91 | 92 | server.send ( 200, "text/html", temp.c_str() ); 93 | digitalWrite ( led, 0 ); 94 | } 95 | 96 | void handleNotFound() { 97 | digitalWrite ( led, 1 ); 98 | String message = "File Not Found\n\n"; 99 | message += "URI: "; 100 | message += server.uri(); 101 | message += "\nMethod: "; 102 | message += ( server.method() == HTTP_GET ) ? "GET" : "POST"; 103 | message += "\nArguments: "; 104 | message += server.args(); 105 | message += "\n"; 106 | 107 | for ( uint8_t i = 0; i < server.args(); i++ ) { 108 | message += " " + server.argName ( i ) + ": " + server.arg ( i ) + "\n"; 109 | } 110 | 111 | server.send ( 404, "text/plain", message ); 112 | digitalWrite ( led, 0 ); 113 | } 114 | 115 | sqlite3 *db1; 116 | int rc; 117 | sqlite3_stmt *res; 118 | int rec_count = 0; 119 | const char *tail; 120 | 121 | int openDb(const char *filename, sqlite3 **db) { 122 | int rc = sqlite3_open(filename, db); 123 | if (rc) { 124 | Serial.printf("Can't open database: %s\n", sqlite3_errmsg(*db)); 125 | return rc; 126 | } else { 127 | Serial.printf("Opened database successfully\n"); 128 | } 129 | return rc; 130 | } 131 | 132 | void setup ( void ) { 133 | pinMode(led, OUTPUT); 134 | digitalWrite(led, 0); 135 | Serial.begin(115200); 136 | WiFi.mode(WIFI_STA); 137 | WiFi.begin(ssid, password); 138 | Serial.println(""); 139 | 140 | // Wait for connection 141 | while ( WiFi.status() != WL_CONNECTED ) { 142 | delay ( 500 ); 143 | Serial.print ( "." ); 144 | } 145 | 146 | Serial.println ( "" ); 147 | Serial.print ( "Connected to " ); 148 | Serial.println ( ssid ); 149 | Serial.print ( "IP address: " ); 150 | Serial.println ( WiFi.localIP() ); 151 | 152 | if ( MDNS.begin ( "esp32" ) ) { 153 | Serial.println ( "MDNS responder started" ); 154 | } 155 | 156 | SD_MMC.begin(); 157 | sqlite3_initialize(); 158 | 159 | // Open database 160 | if (openDb("/sdcard/babyname.db", &db1)) 161 | return; 162 | 163 | server.on ( "/", handleRoot ); 164 | server.on ( "/query_db", []() { 165 | String sql = "Select count(*) from gendered_names where name between '"; 166 | sql += server.arg("from"); 167 | sql += "' and '"; 168 | sql += server.arg("to"); 169 | sql += "'"; 170 | rc = sqlite3_prepare_v2(db1, sql.c_str(), -1, &res, &tail); 171 | if (rc != SQLITE_OK) { 172 | String resp = "Failed to fetch data: "; 173 | resp += sqlite3_errmsg(db1); 174 | resp += ".

"; 175 | server.send ( 200, "text/html", resp.c_str()); 176 | Serial.println(resp.c_str()); 177 | return; 178 | } 179 | while (sqlite3_step(res) == SQLITE_ROW) { 180 | rec_count = sqlite3_column_int(res, 0); 181 | if (rec_count > 5000) { 182 | String resp = "Too many records: "; 183 | resp += rec_count; 184 | resp += ". Please select different range"; 185 | resp += ".

"; 186 | server.send ( 200, "text/html", resp.c_str()); 187 | Serial.println(resp.c_str()); 188 | sqlite3_finalize(res); 189 | return; 190 | } 191 | } 192 | sqlite3_finalize(res); 193 | 194 | sql = "Select year, state, name, total_babies, primary_sex, primary_sex_ratio, per_100k_in_state from gendered_names where name between '"; 195 | sql += server.arg("from"); 196 | sql += "' and '"; 197 | sql += server.arg("to"); 198 | sql += "'"; 199 | rc = sqlite3_prepare_v2(db1, sql.c_str(), -1, &res, &tail); 200 | if (rc != SQLITE_OK) { 201 | String resp = "Failed to fetch data: "; 202 | resp += sqlite3_errmsg(db1); 203 | resp += "

back"; 204 | server.send ( 200, "text/html", resp.c_str()); 205 | Serial.println(resp.c_str()); 206 | return; 207 | } 208 | 209 | rec_count = 0; 210 | server.setContentLength(CONTENT_LENGTH_UNKNOWN); 211 | String resp = "ESP32 Sqlite local database query through web server\ 212 |

ESP32 Sqlite local database query through web server

"; 215 | resp += sql; 216 | resp += "


Statement executed successfully
"; 217 | server.send ( 200, "text/html", resp.c_str()); 218 | while (sqlite3_step(res) == SQLITE_ROW) { 219 | resp = ""; 234 | server.sendContent(resp); 235 | rec_count++; 236 | } 237 | resp = "
YearStateNameTotal babiesPrimary SexRatioPer 100k
"; 220 | resp += sqlite3_column_int(res, 0); 221 | resp += ""; 222 | resp += (const char *) sqlite3_column_text(res, 1); 223 | resp += ""; 224 | resp += (const char *) sqlite3_column_text(res, 2); 225 | resp += ""; 226 | resp += sqlite3_column_int(res, 3); 227 | resp += ""; 228 | resp += (const char *) sqlite3_column_text(res, 4); 229 | resp += ""; 230 | resp += sqlite3_column_double(res, 5); 231 | resp += ""; 232 | resp += sqlite3_column_double(res, 6); 233 | resp += "


Number of records: "; 238 | resp += rec_count; 239 | resp += ".

"; 240 | server.sendContent(resp); 241 | sqlite3_finalize(res); 242 | } ); 243 | server.onNotFound ( handleNotFound ); 244 | server.begin(); 245 | Serial.println ( "HTTP server started" ); 246 | } 247 | 248 | void loop ( void ) { 249 | server.handleClient(); 250 | } 251 | -------------------------------------------------------------------------------- /keywords.txt: -------------------------------------------------------------------------------- 1 | ####################################### 2 | # Syntax Coloring Map SQLite3 3 | ####################################### 4 | 5 | ####################################### 6 | # Datatypes (KEYWORD1) 7 | ####################################### 8 | 9 | sqlite3 KEYWORD1 10 | 11 | ####################################### 12 | # Methods and Functions (KEYWORD2) 13 | ####################################### 14 | 15 | sqlite3_open KEYWORD2 16 | sqlite3_exec KEYWORD2 17 | sqlite3_close KEYWORD2 18 | callback KEYWORD2 19 | 20 | ####################################### 21 | # Constants (LITERAL1) 22 | ####################################### 23 | -------------------------------------------------------------------------------- /lib_mgr_ss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/lib_mgr_ss.png -------------------------------------------------------------------------------- /lib_mgr_ss1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/lib_mgr_ss1.png -------------------------------------------------------------------------------- /library.properties: -------------------------------------------------------------------------------- 1 | name=Sqlite3Esp32 2 | version=2.5 3 | author=Arundale Ramanathan 4 | maintainer=Arun 5 | sentence=Sqlite3 database library for ESP32 core 6 | paragraph=Enables access to Sqlite3 databases from ESP32 (with Shox96 String compression extension). For further information, please visit the given URL. 7 | category=Data Storage 8 | url=https://github.com/siara-cc/esp32_arduino_sqlite3_lib 9 | architectures=esp32 10 | -------------------------------------------------------------------------------- /output_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/output_screenshot.png -------------------------------------------------------------------------------- /output_shox96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/output_shox96.png -------------------------------------------------------------------------------- /output_web_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/output_web_1.png -------------------------------------------------------------------------------- /output_web_1_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/output_web_1_2.png -------------------------------------------------------------------------------- /output_web_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/output_web_2.png -------------------------------------------------------------------------------- /output_web_console.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/output_web_console.png -------------------------------------------------------------------------------- /output_web_so.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/output_web_so.png -------------------------------------------------------------------------------- /output_web_so_id.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/output_web_so_id.png -------------------------------------------------------------------------------- /output_web_so_loc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/output_web_so_loc.png -------------------------------------------------------------------------------- /output_web_so_name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/siara-cc/esp32_arduino_sqlite3_lib/10fb9cfb254cf0261fdb298ee2d7ebc37973f907/output_web_so_name.png -------------------------------------------------------------------------------- /src/config_ext.h: -------------------------------------------------------------------------------- 1 | #define BUILD_sqlite -DNDEBUG 2 | #define SQLITE_OMIT_LOAD_EXTENSION 1 3 | #define SQLITE_DQS 0 4 | #define SQLITE_OS_OTHER 1 5 | #define SQLITE_NO_SYNC 1 6 | #define SQLITE_TEMP_STORE 1 7 | #define SQLITE_DISABLE_LFS 1 8 | #define SQLITE_DISABLE_DIRSYNC 1 9 | #define SQLITE_SECURE_DELETE 0 10 | #define SQLITE_DEFAULT_LOOKASIDE 512,64 11 | #define YYSTACKDEPTH 20 12 | #define SQLITE_SMALL_STACK 1 13 | #define SQLITE_SORTER_PMASZ 4 14 | #define SQLITE_DEFAULT_CACHE_SIZE -1 15 | #define SQLITE_DEFAULT_MEMSTATUS 0 16 | #define SQLITE_DEFAULT_MMAP_SIZE 0 17 | #define SQLITE_CORE 1 18 | #define SQLITE_SYSTEM_MALLOC 1 19 | #define SQLITE_THREADSAFE 0 20 | #define SQLITE_MUTEX_APPDEF 1 21 | #define SQLITE_OMIT_WAL 1 22 | #define SQLITE_DISABLE_FTS3_UNICODE 1 23 | #define SQLITE_DISABLE_FTS4_DEFERRED 1 24 | #define SQLITE_LIKE_DOESNT_MATCH_BLOBS 1 25 | #define SQLITE_DEFAULT_FOREIGN_KEYS 1 26 | #define SQLITE_DEFAULT_LOCKING_MODE 1 27 | #define SQLITE_DEFAULT_PAGE_SIZE 512 28 | #define SQLITE_DEFAULT_PCACHE_INITSZ 8 29 | #define SQLITE_MAX_DEFAULT_PAGE_SIZE 32768 30 | #define SQLITE_POWERSAFE_OVERWRITE 1 31 | #define SQLITE_MAX_EXPR_DEPTH 0 32 | #undef SQLITE_OMIT_FOREIGN_KEY 33 | 34 | /* 35 | #undef SQLITE_OMIT_ALTERTABLE 36 | #undef SQLITE_OMIT_ANALYZE 37 | #undef SQLITE_OMIT_ATTACH 38 | #define SQLITE_OMIT_AUTHORIZATION 1 39 | #undef SQLITE_OMIT_AUTOINCREMENT 40 | #define SQLITE_OMIT_AUTOINIT 1 41 | #define SQLITE_OMIT_AUTOMATIC_INDEX 1 42 | #define SQLITE_OMIT_AUTORESET 1 43 | #define SQLITE_OMIT_AUTOVACUUM 1 44 | #undef SQLITE_OMIT_BETWEEN_OPTIMIZATION 45 | #define SQLITE_OMIT_BLOB_LITERAL 1 46 | #define SQLITE_OMIT_BTREECOUNT 1 47 | #define SQLITE_OMIT_BUILTIN_TEST 1 48 | #define SQLITE_OMIT_CAST 1 49 | #define SQLITE_OMIT_CHECK 1 50 | #define SQLITE_OMIT_COMPILEOPTION_DIAGS 1 51 | #define SQLITE_OMIT_COMPOUND_SELECT 1 52 | #define SQLITE_OMIT_CONFLICT_CLAUSE 1 53 | #undef SQLITE_OMIT_CTE 54 | #define SQLITE_OMIT_DECLTYPE 1 55 | #define SQLITE_OMIT_DEPRECATED 1 56 | #undef SQLITE_OMIT_DISKIO 57 | #define SQLITE_OMIT_EXPLAIN 1 58 | #define SQLITE_OMIT_FLAG_PRAGMAS 1 59 | #define SQLITE_OMIT_FOREIGN_KEY 1 60 | #define SQLITE_OMIT_GET_TABLE 1 61 | #define SQLITE_OMIT_INCRBLOB 1 62 | #define SQLITE_OMIT_INTEGRITY_CHECK 1 63 | #undef SQLITE_OMIT_LIKE_OPTIMIZATION 64 | #undef SQLITE_OMIT_LOCALTIME 65 | #define SQLITE_OMIT_LOOKASIDE 1 66 | #undef SQLITE_OMIT_MEMORYDB 67 | #undef SQLITE_OMIT_OR_OPTIMIZATION 68 | #undef SQLITE_OMIT_PAGER_PRAGMAS 69 | #define SQLITE_OMIT_PARSER_TRACE 1 70 | #undef SQLITE_OMIT_PRAGMA 71 | #define SQLITE_OMIT_PROGRESS_CALLBACK 1 72 | #define SQLITE_OMIT_QUICKBALANCE 1 73 | #undef SQLITE_OMIT_REINDEX 74 | #define SQLITE_OMIT_SCHEMA_PRAGMAS 1 75 | #define SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS 1 76 | #define SQLITE_OMIT_SHARED_CACHE 1 77 | #define SQLITE_OMIT_TCL_VARIABLE 1 78 | #define SQLITE_OMIT_TEMPDB 1 79 | #define SQLITE_OMIT_TRACE 1 80 | #undef SQLITE_OMIT_TRIGGER 81 | #define SQLITE_OMIT_TRUNCATE_OPTIMIZATION 1 82 | #define SQLITE_OMIT_UTF16 1 83 | #undef SQLITE_OMIT_VACUUM 84 | #undef SQLITE_OMIT_VIEW 85 | #undef SQLITE_OMIT_VIRTUALTABLE 86 | #undef SQLITE_OMIT_WSD 87 | #define SQLITE_OMIT_XFER_OPT 1 88 | #define SQLITE_PERFORMANCE_TRACE 1 89 | //#define SQLITE_OMIT_COMPLETE 1 90 | //#define SQLITE_OMIT_SUBQUERY 1 91 | //#define SQLITE_OMIT_DATETIME_FUNCS 1 92 | //#define SQLITE_OMIT_FLOATING_POINT 1 93 | #define SQLITE_COUNTOFVIEW_OPTIMIZATION 0 94 | */ 95 | -------------------------------------------------------------------------------- /src/esp32.cpp: -------------------------------------------------------------------------------- 1 | /* From: https://chromium.googlesource.com/chromium/src.git/+/4.1.249.1050/third_party/sqlite/src/os_symbian.cc 2 | * https://github.com/spsoft/spmemvfs/tree/master/spmemvfs 3 | * http://www.sqlite.org/src/doc/trunk/src/test_ESP32vfs.c 4 | * http://www.sqlite.org/src/doc/trunk/src/test_vfstrace.c 5 | * http://www.sqlite.org/src/doc/trunk/src/test_onefile.c 6 | * http://www.sqlite.org/src/doc/trunk/src/test_vfs.c 7 | * https://github.com/nodemcu/nodemcu-firmware/blob/master/app/sqlite3/esp8266.c 8 | **/ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include "shox96_0_2.h" 27 | 28 | #undef dbg_printf 29 | //#define dbg_printf(...) Serial.printf(__VA_ARGS__) 30 | #define dbg_printf(...) 0 31 | 32 | extern "C" { 33 | void SerialPrintln(const char *str) { 34 | //Serial.println(str); 35 | } 36 | } 37 | 38 | // From https://stackoverflow.com/questions/19758270/read-varint-from-linux-sockets#19760246 39 | // Encode an unsigned 64-bit varint. Returns number of encoded bytes. 40 | // 'buffer' must have room for up to 10 bytes. 41 | int encode_unsigned_varint(uint8_t *buffer, uint64_t value) { 42 | int encoded = 0; 43 | do { 44 | uint8_t next_byte = value & 0x7F; 45 | value >>= 7; 46 | if (value) 47 | next_byte |= 0x80; 48 | buffer[encoded++] = next_byte; 49 | } while (value); 50 | return encoded; 51 | } 52 | 53 | uint64_t decode_unsigned_varint(const uint8_t *data, int &decoded_bytes) { 54 | int i = 0; 55 | uint64_t decoded_value = 0; 56 | int shift_amount = 0; 57 | do { 58 | decoded_value |= (uint64_t)(data[i] & 0x7F) << shift_amount; 59 | shift_amount += 7; 60 | } while ((data[i++] & 0x80) != 0); 61 | decoded_bytes = i; 62 | return decoded_value; 63 | } 64 | 65 | /* 66 | ** Size of the write buffer used by journal files in bytes. 67 | */ 68 | #ifndef SQLITE_ESP32VFS_BUFFERSZ 69 | # define SQLITE_ESP32VFS_BUFFERSZ 8192 70 | #endif 71 | 72 | /* 73 | ** The maximum pathname length supported by this VFS. 74 | */ 75 | #define MAXPATHNAME 100 76 | 77 | /* 78 | ** When using this VFS, the sqlite3_file* handles that SQLite uses are 79 | ** actually pointers to instances of type ESP32File. 80 | */ 81 | typedef struct ESP32File ESP32File; 82 | struct ESP32File { 83 | sqlite3_file base; /* Base class. Must be first. */ 84 | FILE *fp; /* File descriptor */ 85 | 86 | char *aBuffer; /* Pointer to malloc'd buffer */ 87 | int nBuffer; /* Valid bytes of data in zBuffer */ 88 | sqlite3_int64 iBufferOfst; /* Offset in file of zBuffer[0] */ 89 | }; 90 | 91 | /* 92 | ** Write directly to the file passed as the first argument. Even if the 93 | ** file has a write-buffer (ESP32File.aBuffer), ignore it. 94 | */ 95 | static int ESP32DirectWrite( 96 | ESP32File *p, /* File handle */ 97 | const void *zBuf, /* Buffer containing data to write */ 98 | int iAmt, /* Size of data to write in bytes */ 99 | sqlite_int64 iOfst /* File offset to write to */ 100 | ){ 101 | off_t ofst; /* Return value from lseek() */ 102 | size_t nWrite; /* Return value from write() */ 103 | 104 | //Serial.println("fn: DirectWrite:"); 105 | 106 | ofst = fseek(p->fp, iOfst, SEEK_SET); //lseek(p->fd, iOfst, SEEK_SET); 107 | if( ofst != 0 ){ 108 | //Serial.println("Seek error"); 109 | return SQLITE_IOERR_WRITE; 110 | } 111 | 112 | nWrite = fwrite(zBuf, 1, iAmt, p->fp); // write(p->fd, zBuf, iAmt); 113 | if( nWrite!=iAmt ){ 114 | //Serial.println("Write error"); 115 | return SQLITE_IOERR_WRITE; 116 | } 117 | 118 | //Serial.println("fn:DirectWrite:Success"); 119 | 120 | return SQLITE_OK; 121 | } 122 | 123 | /* 124 | ** Flush the contents of the ESP32File.aBuffer buffer to disk. This is a 125 | ** no-op if this particular file does not have a buffer (i.e. it is not 126 | ** a journal file) or if the buffer is currently empty. 127 | */ 128 | static int ESP32FlushBuffer(ESP32File *p){ 129 | int rc = SQLITE_OK; 130 | //Serial.println("fn: FlushBuffer"); 131 | if( p->nBuffer ){ 132 | rc = ESP32DirectWrite(p, p->aBuffer, p->nBuffer, p->iBufferOfst); 133 | p->nBuffer = 0; 134 | } 135 | //Serial.println("fn:FlushBuffer:Success"); 136 | return rc; 137 | } 138 | 139 | /* 140 | ** Close a file. 141 | */ 142 | static int ESP32Close(sqlite3_file *pFile){ 143 | int rc; 144 | //Serial.println("fn: Close"); 145 | ESP32File *p = (ESP32File*)pFile; 146 | rc = ESP32FlushBuffer(p); 147 | sqlite3_free(p->aBuffer); 148 | fclose(p->fp); 149 | //Serial.println("fn:Close:Success"); 150 | return rc; 151 | } 152 | 153 | /* 154 | ** Read data from a file. 155 | */ 156 | static int ESP32Read( 157 | sqlite3_file *pFile, 158 | void *zBuf, 159 | int iAmt, 160 | sqlite_int64 iOfst 161 | ){ 162 | //Serial.println("fn: Read"); 163 | ESP32File *p = (ESP32File*)pFile; 164 | off_t ofst; /* Return value from lseek() */ 165 | int nRead; /* Return value from read() */ 166 | int rc; /* Return code from ESP32FlushBuffer() */ 167 | 168 | /* Flush any data in the write buffer to disk in case this operation 169 | ** is trying to read data the file-region currently cached in the buffer. 170 | ** It would be possible to detect this case and possibly save an 171 | ** unnecessary write here, but in practice SQLite will rarely read from 172 | ** a journal file when there is data cached in the write-buffer. 173 | */ 174 | rc = ESP32FlushBuffer(p); 175 | if( rc!=SQLITE_OK ){ 176 | return rc; 177 | } 178 | 179 | ofst = fseek(p->fp, iOfst, SEEK_SET); //lseek(p->fd, iOfst, SEEK_SET); 180 | //if( ofst != 0 ){ 181 | // return SQLITE_IOERR_READ; 182 | //} 183 | nRead = fread(zBuf, 1, iAmt, p->fp); // read(p->fd, zBuf, iAmt); 184 | 185 | if( nRead==iAmt ){ 186 | //Serial.println("fn:Read:Success"); 187 | return SQLITE_OK; 188 | }else if( nRead>=0 ){ 189 | return SQLITE_IOERR_SHORT_READ; 190 | } 191 | 192 | return SQLITE_IOERR_READ; 193 | } 194 | 195 | /* 196 | ** Write data to a crash-file. 197 | */ 198 | static int ESP32Write( 199 | sqlite3_file *pFile, 200 | const void *zBuf, 201 | int iAmt, 202 | sqlite_int64 iOfst 203 | ){ 204 | //Serial.println("fn: Write"); 205 | ESP32File *p = (ESP32File*)pFile; 206 | 207 | if( p->aBuffer ){ 208 | char *z = (char *)zBuf; /* Pointer to remaining data to write */ 209 | int n = iAmt; /* Number of bytes at z */ 210 | sqlite3_int64 i = iOfst; /* File offset to write to */ 211 | 212 | while( n>0 ){ 213 | int nCopy; /* Number of bytes to copy into buffer */ 214 | 215 | /* If the buffer is full, or if this data is not being written directly 216 | ** following the data already buffered, flush the buffer. Flushing 217 | ** the buffer is a no-op if it is empty. 218 | */ 219 | if( p->nBuffer==SQLITE_ESP32VFS_BUFFERSZ || p->iBufferOfst+p->nBuffer!=i ){ 220 | int rc = ESP32FlushBuffer(p); 221 | if( rc!=SQLITE_OK ){ 222 | return rc; 223 | } 224 | } 225 | assert( p->nBuffer==0 || p->iBufferOfst+p->nBuffer==i ); 226 | p->iBufferOfst = i - p->nBuffer; 227 | 228 | /* Copy as much data as possible into the buffer. */ 229 | nCopy = SQLITE_ESP32VFS_BUFFERSZ - p->nBuffer; 230 | if( nCopy>n ){ 231 | nCopy = n; 232 | } 233 | memcpy(&p->aBuffer[p->nBuffer], z, nCopy); 234 | p->nBuffer += nCopy; 235 | 236 | n -= nCopy; 237 | i += nCopy; 238 | z += nCopy; 239 | } 240 | }else{ 241 | return ESP32DirectWrite(p, zBuf, iAmt, iOfst); 242 | } 243 | //Serial.println("fn:Write:Success"); 244 | 245 | return SQLITE_OK; 246 | } 247 | 248 | /* 249 | ** Truncate a file. This is a no-op for this VFS (see header comments at 250 | ** the top of the file). 251 | */ 252 | static int ESP32Truncate(sqlite3_file *pFile, sqlite_int64 size){ 253 | //Serial.println("fn: Truncate"); 254 | #if 0 255 | if( ftruncate(((ESP32File *)pFile)->fd, size) ) return SQLITE_IOERR_TRUNCATE; 256 | #endif 257 | //Serial.println("fn:Truncate:Success"); 258 | return SQLITE_OK; 259 | } 260 | 261 | /* 262 | ** Sync the contents of the file to the persistent media. 263 | */ 264 | static int ESP32Sync(sqlite3_file *pFile, int flags){ 265 | //Serial.println("fn: Sync"); 266 | ESP32File *p = (ESP32File*)pFile; 267 | int rc; 268 | 269 | rc = ESP32FlushBuffer(p); 270 | if( rc!=SQLITE_OK ){ 271 | return rc; 272 | } 273 | rc = fflush(p->fp); 274 | if (rc != 0) 275 | return SQLITE_IOERR_FSYNC; 276 | rc = fsync(fileno(p->fp)); 277 | //if (rc == 0) 278 | //Serial.println("fn:Sync:Success"); 279 | return SQLITE_OK; // ignore fsync return value // (rc==0 ? SQLITE_OK : SQLITE_IOERR_FSYNC); 280 | } 281 | 282 | /* 283 | ** Write the size of the file in bytes to *pSize. 284 | */ 285 | static int ESP32FileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ 286 | //Serial.println("fn: FileSize"); 287 | ESP32File *p = (ESP32File*)pFile; 288 | int rc; /* Return code from fstat() call */ 289 | struct stat sStat; /* Output of fstat() call */ 290 | 291 | /* Flush the contents of the buffer to disk. As with the flush in the 292 | ** ESP32Read() method, it would be possible to avoid this and save a write 293 | ** here and there. But in practice this comes up so infrequently it is 294 | ** not worth the trouble. 295 | */ 296 | rc = ESP32FlushBuffer(p); 297 | if( rc!=SQLITE_OK ){ 298 | return rc; 299 | } 300 | 301 | struct stat st; 302 | int fno = fileno(p->fp); 303 | if (fno < 0) 304 | return SQLITE_IOERR_FSTAT; 305 | if (fstat(fno, &st)) 306 | return SQLITE_IOERR_FSTAT; 307 | *pSize = st.st_size; 308 | //Serial.println("fn:FileSize:Success"); 309 | return SQLITE_OK; 310 | } 311 | 312 | /* 313 | ** Locking functions. The xLock() and xUnlock() methods are both no-ops. 314 | ** The xCheckReservedLock() always indicates that no other process holds 315 | ** a reserved lock on the database file. This ensures that if a hot-journal 316 | ** file is found in the file-system it is rolled back. 317 | */ 318 | static int ESP32Lock(sqlite3_file *pFile, int eLock){ 319 | return SQLITE_OK; 320 | } 321 | static int ESP32Unlock(sqlite3_file *pFile, int eLock){ 322 | return SQLITE_OK; 323 | } 324 | static int ESP32CheckReservedLock(sqlite3_file *pFile, int *pResOut){ 325 | *pResOut = 0; 326 | return SQLITE_OK; 327 | } 328 | 329 | /* 330 | ** No xFileControl() verbs are implemented by this VFS. 331 | */ 332 | static int ESP32FileControl(sqlite3_file *pFile, int op, void *pArg){ 333 | return SQLITE_OK; 334 | } 335 | 336 | /* 337 | ** The xSectorSize() and xDeviceCharacteristics() methods. These two 338 | ** may return special values allowing SQLite to optimize file-system 339 | ** access to some extent. But it is also safe to simply return 0. 340 | */ 341 | static int ESP32SectorSize(sqlite3_file *pFile){ 342 | return 0; 343 | } 344 | static int ESP32DeviceCharacteristics(sqlite3_file *pFile){ 345 | return 0; 346 | } 347 | 348 | #ifndef F_OK 349 | # define F_OK 0 350 | #endif 351 | #ifndef R_OK 352 | # define R_OK 4 353 | #endif 354 | #ifndef W_OK 355 | # define W_OK 2 356 | #endif 357 | 358 | /* 359 | ** Query the file-system to see if the named file exists, is readable or 360 | ** is both readable and writable. 361 | */ 362 | static int ESP32Access( 363 | sqlite3_vfs *pVfs, 364 | const char *zPath, 365 | int flags, 366 | int *pResOut 367 | ){ 368 | int rc; /* access() return code */ 369 | int eAccess = F_OK; /* Second argument to access() */ 370 | //Serial.println("fn: Access"); 371 | 372 | assert( flags==SQLITE_ACCESS_EXISTS /* access(zPath, F_OK) */ 373 | || flags==SQLITE_ACCESS_READ /* access(zPath, R_OK) */ 374 | || flags==SQLITE_ACCESS_READWRITE /* access(zPath, R_OK|W_OK) */ 375 | ); 376 | 377 | if( flags==SQLITE_ACCESS_READWRITE ) eAccess = R_OK|W_OK; 378 | if( flags==SQLITE_ACCESS_READ ) eAccess = R_OK; 379 | 380 | rc = access(zPath, eAccess); 381 | *pResOut = (rc==0); 382 | //Serial.println("fn:Access:Success"); 383 | return SQLITE_OK; 384 | } 385 | 386 | static char dbrootpath[MAXPATHNAME+1]; 387 | 388 | /* 389 | ** Open a file handle. 390 | */ 391 | static int ESP32Open( 392 | sqlite3_vfs *pVfs, /* VFS */ 393 | const char *zName, /* File to open, or 0 for a temp file */ 394 | sqlite3_file *pFile, /* Pointer to ESP32File struct to populate */ 395 | int flags, /* Input SQLITE_OPEN_XXX flags */ 396 | int *pOutFlags /* Output SQLITE_OPEN_XXX flags (or NULL) */ 397 | ){ 398 | static const sqlite3_io_methods ESP32io = { 399 | 1, /* iVersion */ 400 | ESP32Close, /* xClose */ 401 | ESP32Read, /* xRead */ 402 | ESP32Write, /* xWrite */ 403 | ESP32Truncate, /* xTruncate */ 404 | ESP32Sync, /* xSync */ 405 | ESP32FileSize, /* xFileSize */ 406 | ESP32Lock, /* xLock */ 407 | ESP32Unlock, /* xUnlock */ 408 | ESP32CheckReservedLock, /* xCheckReservedLock */ 409 | ESP32FileControl, /* xFileControl */ 410 | ESP32SectorSize, /* xSectorSize */ 411 | ESP32DeviceCharacteristics /* xDeviceCharacteristics */ 412 | }; 413 | 414 | ESP32File *p = (ESP32File*)pFile; /* Populate this structure */ 415 | int oflags = 0; /* flags to pass to open() call */ 416 | char *aBuf = 0; 417 | char mode[5]; 418 | //Serial.println("fn: Open"); 419 | 420 | strcpy(mode, "r"); 421 | 422 | if( flags&SQLITE_OPEN_MAIN_JOURNAL ){ 423 | aBuf = (char *)sqlite3_malloc(SQLITE_ESP32VFS_BUFFERSZ); 424 | if( !aBuf ){ 425 | return SQLITE_NOMEM; 426 | } 427 | } 428 | 429 | if( flags&SQLITE_OPEN_CREATE || flags&SQLITE_OPEN_READWRITE 430 | || flags&SQLITE_OPEN_MAIN_JOURNAL ) { 431 | struct stat st; 432 | memset(&st, 0, sizeof(struct stat)); 433 | int rc = (zName == 0 ? -1 : stat( zName, &st )); 434 | //Serial.println(zName); 435 | if (rc < 0) { 436 | strcpy(mode, "w+"); 437 | //int fd = open(zName, (O_CREAT | O_RDWR | O_EXCL), S_IRUSR | S_IWUSR); 438 | //close(fd); 439 | //oflags |= (O_CREAT | O_RDWR); 440 | //Serial.println("Create mode"); 441 | } else 442 | strcpy(mode, "r+"); 443 | } 444 | 445 | memset(p, 0, sizeof(ESP32File)); 446 | //p->fd = open(zName, oflags, 0600); 447 | //p->fd = open(zName, oflags, S_IRUSR | S_IWUSR); 448 | if (zName == 0) { 449 | //generate a temporary file name 450 | char *tName = tmpnam(NULL); 451 | tName[4] = '_'; 452 | size_t len = strlen(dbrootpath); 453 | memmove(tName + len, tName, strlen(tName) + 1); 454 | memcpy(tName, dbrootpath, len); 455 | p->fp = fopen(tName, mode); 456 | //https://stackoverflow.com/questions/64424287/how-to-delete-a-file-in-c-using-a-file-descriptor 457 | //for temp file, then no need to handle in esp32close 458 | unlink(tName); 459 | //Serial.println("Temporary file name generated: " + String(tName) + " mode: " + String(mode)); 460 | } else { 461 | //detect database root as folder for temporary files, every newly openened db will change this path 462 | //this mainly fixes that vfs's have their own root name like /sd 463 | char *ext = strrchr(zName, '.'); 464 | bool isdb = false; 465 | if (ext) { 466 | isdb = (strcmp(ext+1,"db") == 0); 467 | } 468 | if (isdb) { 469 | char zDir[MAXPATHNAME+1]; 470 | int i=0; 471 | strcpy(zDir,zName); 472 | 473 | for(i=1; zDir[i]!='/'; i++) {}; 474 | zDir[i] = '\0'; 475 | 476 | strcpy(dbrootpath, zDir); 477 | } 478 | 479 | p->fp = fopen(zName, mode); 480 | } 481 | 482 | if( p->fp == NULL){ 483 | if (aBuf) 484 | sqlite3_free(aBuf); 485 | //Serial.println("Can't open"); 486 | return SQLITE_CANTOPEN; 487 | } 488 | p->aBuffer = aBuf; 489 | 490 | if( pOutFlags ){ 491 | *pOutFlags = flags; 492 | } 493 | p->base.pMethods = &ESP32io; 494 | //Serial.println("fn:Open:Success"); 495 | return SQLITE_OK; 496 | } 497 | 498 | /* 499 | ** Delete the file identified by argument zPath. If the dirSync parameter 500 | ** is non-zero, then ensure the file-system modification to delete the 501 | ** file has been synced to disk before returning. 502 | */ 503 | static int ESP32Delete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ 504 | int rc; /* Return code */ 505 | 506 | //Serial.println("fn: Delete"); 507 | 508 | rc = unlink(zPath); 509 | if( rc!=0 && errno==ENOENT ) return SQLITE_OK; 510 | 511 | if( rc==0 && dirSync ){ 512 | FILE *dfd; /* File descriptor open on directory */ 513 | int i; /* Iterator variable */ 514 | char zDir[MAXPATHNAME+1]; /* Name of directory containing file zPath */ 515 | 516 | /* Figure out the directory name from the path of the file deleted. */ 517 | sqlite3_snprintf(MAXPATHNAME, zDir, "%s", zPath); 518 | zDir[MAXPATHNAME] = '\0'; 519 | for(i=strlen(zDir); i>1 && zDir[i]!='/'; i++); 520 | zDir[i] = '\0'; 521 | 522 | /* Open a file-descriptor on the directory. Sync. Close. */ 523 | dfd = fopen(zDir, "r"); 524 | if( dfd == (void *)NULL ){ 525 | rc = -1; 526 | }else{ 527 | rc = fflush(dfd); 528 | rc = fsync(fileno(dfd)); 529 | fclose(dfd); 530 | } 531 | } 532 | //if (rc == 0) 533 | //Serial.println("fn:Delete:Success"); 534 | return (rc==0 ? SQLITE_OK : SQLITE_IOERR_DELETE); 535 | } 536 | 537 | /* 538 | ** Argument zPath points to a nul-terminated string containing a file path. 539 | ** If zPath is an absolute path, then it is copied as is into the output 540 | ** buffer. Otherwise, if it is a relative path, then the equivalent full 541 | ** path is written to the output buffer. 542 | ** 543 | ** This function assumes that paths are UNIX style. Specifically, that: 544 | ** 545 | ** 1. Path components are separated by a '/'. and 546 | ** 2. Full paths begin with a '/' character. 547 | */ 548 | static int ESP32FullPathname( 549 | sqlite3_vfs *pVfs, /* VFS */ 550 | const char *zPath, /* Input path (possibly a relative path) */ 551 | int nPathOut, /* Size of output buffer in bytes */ 552 | char *zPathOut /* Pointer to output buffer */ 553 | ){ 554 | //Serial.print("fn: FullPathName"); 555 | //char zDir[MAXPATHNAME+1]; 556 | //if( zPath[0]=='/' ){ 557 | // zDir[0] = '\0'; 558 | //}else{ 559 | // if( getcwd(zDir, sizeof(zDir))==0 ) return SQLITE_IOERR; 560 | //} 561 | //zDir[MAXPATHNAME] = '\0'; 562 | strncpy( zPathOut, zPath, nPathOut ); 563 | 564 | //sqlite3_snprintf(nPathOut, zPathOut, "%s/%s", zDir, zPath); 565 | zPathOut[nPathOut-1] = '\0'; 566 | //Serial.println("fn:Fullpathname:Success"); 567 | 568 | return SQLITE_OK; 569 | } 570 | 571 | /* 572 | ** The following four VFS methods: 573 | ** 574 | ** xDlOpen 575 | ** xDlError 576 | ** xDlSym 577 | ** xDlClose 578 | ** 579 | ** are supposed to implement the functionality needed by SQLite to load 580 | ** extensions compiled as shared objects. This simple VFS does not support 581 | ** this functionality, so the following functions are no-ops. 582 | */ 583 | static void *ESP32DlOpen(sqlite3_vfs *pVfs, const char *zPath){ 584 | return 0; 585 | } 586 | static void ESP32DlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ 587 | sqlite3_snprintf(nByte, zErrMsg, "Loadable extensions are not supported"); 588 | zErrMsg[nByte-1] = '\0'; 589 | } 590 | static void (*ESP32DlSym(sqlite3_vfs *pVfs, void *pH, const char *z))(void){ 591 | return 0; 592 | } 593 | static void ESP32DlClose(sqlite3_vfs *pVfs, void *pHandle){ 594 | return; 595 | } 596 | 597 | /* 598 | ** Parameter zByte points to a buffer nByte bytes in size. Populate this 599 | ** buffer with pseudo-random data. 600 | */ 601 | static int ESP32Randomness(sqlite3_vfs *pVfs, int nByte, char *zByte){ 602 | return SQLITE_OK; 603 | } 604 | 605 | /* 606 | ** Sleep for at least nMicro microseconds. Return the (approximate) number 607 | ** of microseconds slept for. 608 | */ 609 | static int ESP32Sleep(sqlite3_vfs *pVfs, int nMicro){ 610 | sleep(nMicro / 1000000); 611 | usleep(nMicro % 1000000); 612 | return nMicro; 613 | } 614 | 615 | /* 616 | ** Set *pTime to the current UTC time expressed as a Julian day. Return 617 | ** SQLITE_OK if successful, or an error code otherwise. 618 | ** 619 | ** http://en.wikipedia.org/wiki/Julian_day 620 | ** 621 | ** This implementation is not very good. The current time is rounded to 622 | ** an integer number of seconds. Also, assuming time_t is a signed 32-bit 623 | ** value, it will stop working some time in the year 2038 AD (the so-called 624 | ** "year 2038" problem that afflicts systems that store time this way). 625 | */ 626 | static int ESP32CurrentTime(sqlite3_vfs *pVfs, double *pTime){ 627 | time_t t = time(0); 628 | *pTime = t/86400.0 + 2440587.5; 629 | return SQLITE_OK; 630 | } 631 | 632 | /* 633 | ** This function returns a pointer to the VFS implemented in this file. 634 | ** To make the VFS available to SQLite: 635 | ** 636 | ** sqlite3_vfs_register(sqlite3_ESP32vfs(), 0); 637 | */ 638 | sqlite3_vfs *sqlite3_ESP32vfs(void){ 639 | static sqlite3_vfs ESP32vfs = { 640 | 1, // iVersion 641 | sizeof(ESP32File), // szOsFile 642 | MAXPATHNAME, // mxPathname 643 | 0, // pNext 644 | "ESP32", // zName 645 | 0, // pAppData 646 | ESP32Open, // xOpen 647 | ESP32Delete, // xDelete 648 | ESP32Access, // xAccess 649 | ESP32FullPathname, // xFullPathname 650 | ESP32DlOpen, // xDlOpen 651 | ESP32DlError, // xDlError 652 | ESP32DlSym, // xDlSym 653 | ESP32DlClose, // xDlClose 654 | ESP32Randomness, // xRandomness 655 | ESP32Sleep, // xSleep 656 | ESP32CurrentTime, // xCurrentTime 657 | }; 658 | return &ESP32vfs; 659 | } 660 | 661 | static void shox96_0_2c(sqlite3_context *context, int argc, sqlite3_value **argv) { 662 | int nIn, nOut; 663 | long int nOut2; 664 | const unsigned char *inBuf; 665 | unsigned char *outBuf; 666 | unsigned char vInt[9]; 667 | int vIntLen; 668 | 669 | assert( argc==1 ); 670 | nIn = sqlite3_value_bytes(argv[0]); 671 | inBuf = (unsigned char *) sqlite3_value_blob(argv[0]); 672 | nOut = 13 + nIn + (nIn+999)/1000; 673 | vIntLen = encode_unsigned_varint(vInt, (uint64_t) nIn); 674 | 675 | outBuf = (unsigned char *) malloc( nOut+vIntLen ); 676 | memcpy(outBuf, vInt, vIntLen); 677 | nOut2 = shox96_0_2_compress((const char *) inBuf, nIn, (char *) &outBuf[vIntLen], NULL); 678 | sqlite3_result_blob(context, outBuf, nOut2+vIntLen, free); 679 | } 680 | 681 | static void shox96_0_2d(sqlite3_context *context, int argc, sqlite3_value **argv) { 682 | unsigned int nIn, nOut, rc; 683 | const unsigned char *inBuf; 684 | unsigned char *outBuf; 685 | long int nOut2; 686 | uint64_t inBufLen64; 687 | int vIntLen; 688 | 689 | assert( argc==1 ); 690 | 691 | if (sqlite3_value_type(argv[0]) != SQLITE_BLOB) 692 | return; 693 | 694 | nIn = sqlite3_value_bytes(argv[0]); 695 | if (nIn < 2){ 696 | return; 697 | } 698 | inBuf = (unsigned char *) sqlite3_value_blob(argv[0]); 699 | inBufLen64 = decode_unsigned_varint(inBuf, vIntLen); 700 | nOut = (unsigned int) inBufLen64; 701 | outBuf = (unsigned char *) malloc( nOut ); 702 | //nOut2 = (long int)nOut; 703 | nOut2 = shox96_0_2_decompress((const char *) (inBuf + vIntLen), nIn - vIntLen, (char *) outBuf, NULL); 704 | //if( rc!=Z_OK ){ 705 | // free(outBuf); 706 | //}else{ 707 | sqlite3_result_blob(context, outBuf, nOut2, free); 708 | //} 709 | } 710 | 711 | static void unishox1c(sqlite3_context *context, int argc, sqlite3_value **argv) { 712 | int nIn, nOut; 713 | long int nOut2; 714 | const unsigned char *inBuf; 715 | unsigned char *outBuf; 716 | unsigned char vInt[9]; 717 | int vIntLen; 718 | 719 | assert( argc==1 ); 720 | nIn = sqlite3_value_bytes(argv[0]); 721 | inBuf = (unsigned char *) sqlite3_value_blob(argv[0]); 722 | nOut = 13 + nIn + (nIn+999)/1000; 723 | vIntLen = encode_unsigned_varint(vInt, (uint64_t) nIn); 724 | 725 | outBuf = (unsigned char *) malloc( nOut+vIntLen ); 726 | memcpy(outBuf, vInt, vIntLen); 727 | nOut2 = shox96_0_2_compress((const char *) inBuf, nIn, (char *) &outBuf[vIntLen], NULL); 728 | sqlite3_result_blob(context, outBuf, nOut2+vIntLen, free); 729 | } 730 | 731 | static void unishox1d(sqlite3_context *context, int argc, sqlite3_value **argv) { 732 | unsigned int nIn, nOut, rc; 733 | const unsigned char *inBuf; 734 | unsigned char *outBuf; 735 | long int nOut2; 736 | uint64_t inBufLen64; 737 | int vIntLen; 738 | 739 | assert( argc==1 ); 740 | 741 | if (sqlite3_value_type(argv[0]) != SQLITE_BLOB) 742 | return; 743 | 744 | nIn = sqlite3_value_bytes(argv[0]); 745 | if (nIn < 2){ 746 | return; 747 | } 748 | inBuf = (unsigned char *) sqlite3_value_blob(argv[0]); 749 | inBufLen64 = decode_unsigned_varint(inBuf, vIntLen); 750 | nOut = (unsigned int) inBufLen64; 751 | outBuf = (unsigned char *) malloc( nOut ); 752 | //nOut2 = (long int)nOut; 753 | nOut2 = shox96_0_2_decompress((const char *) (inBuf + vIntLen), nIn - vIntLen, (char *) outBuf, NULL); 754 | //if( rc!=Z_OK ){ 755 | // free(outBuf); 756 | //}else{ 757 | sqlite3_result_blob(context, outBuf, nOut2, free); 758 | //} 759 | } 760 | 761 | int registerFunctions(sqlite3 *db, const char **pzErrMsg, const struct sqlite3_api_routines *pThunk) { 762 | sqlite3_create_function(db, "shox96_0_2c", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, shox96_0_2c, 0, 0); 763 | sqlite3_create_function(db, "shox96_0_2d", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, shox96_0_2d, 0, 0); 764 | sqlite3_create_function(db, "unishox1c", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, unishox1c, 0, 0); 765 | sqlite3_create_function(db, "unishox1d", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, unishox1d, 0, 0); 766 | return SQLITE_OK; 767 | } 768 | 769 | void errorLogCallback(void *pArg, int iErrCode, const char *zMsg) { 770 | //Serial.printf("(%d) %s\n", iErrCode, zMsg); 771 | } 772 | 773 | int sqlite3_os_init(void){ 774 | //sqlite3_config(SQLITE_CONFIG_LOG, errorLogCallback, NULL); 775 | sqlite3_vfs_register(sqlite3_ESP32vfs(), 1); 776 | sqlite3_auto_extension((void (*)())registerFunctions); 777 | return SQLITE_OK; 778 | } 779 | 780 | int sqlite3_os_end(void){ 781 | return SQLITE_OK; 782 | } 783 | -------------------------------------------------------------------------------- /src/shox96_0_2.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2019 Siara Logics (cc) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * @author Arundale R. 17 | * 18 | */ 19 | #pragma GCC diagnostic push 20 | #pragma GCC diagnostic ignored "-Wunused-value" 21 | #pragma GCC diagnostic ignored "-Wunused-label" 22 | #pragma GCC diagnostic ignored "-Wchar-subscripts" 23 | #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #include "shox96_0_2.h" 33 | 34 | typedef unsigned char byte; 35 | 36 | static unsigned int c_95[95] = {16384, 16256, 15744, 16192, 15328, 15344, 15360, 16064, 15264, 15296, 15712, 15200, 14976, 15040, 14848, 15104, 14528, 14592, 14656, 14688, 14720, 14752, 14784, 14816, 14832, 14464, 15552, 15488, 15616, 15168, 15680, 16000, 15872, 10752, 8576, 8192, 8320, 9728, 8672, 8608, 8384, 11264, 9024, 8992, 12160, 8544, 11520, 11008, 8512, 9008, 12032, 11776, 10240, 8448, 8960, 8640, 9040, 8688, 9048, 15840, 16288, 15856, 16128, 16224, 16368, 40960, 6144, 0, 2048, 24576, 7680, 6656, 3072, 49152, 13312, 12800, 63488, 5632, 53248, 45056, 5120, 13056, 61440, 57344, 32768, 4096, 12288, 7168, 13568, 7936, 13696, 15776, 16320, 15808, 16352}; 37 | static unsigned char l_95[95] = { 3, 11, 11, 11, 12, 12, 9, 10, 11, 11, 11, 11, 10, 10, 9, 10, 10, 10, 11, 11, 11, 11, 11, 12, 12, 10, 10, 10, 10, 11, 11, 10, 9, 8, 11, 9, 10, 7, 12, 11, 10, 8, 12, 12, 9, 11, 8, 8, 11, 12, 9, 8, 7, 10, 11, 11, 13, 12, 13, 12, 11, 12, 10, 11, 12, 4, 7, 5, 6, 3, 8, 7, 6, 4, 8, 8, 5, 7, 4, 4, 7, 8, 5, 4, 3, 6, 7, 7, 9, 8, 9, 11, 11, 11, 12}; 38 | //unsigned char c[] = { ' ', '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~'}; 39 | static char SET2_STR[] = {'9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '.', ',', '-', '/', '=', '+', ' ', '(', ')', '$', '%', '&', ';', ':', '<', '>', '*', '"', '{', '}', '[', ']', '@', '?', '\'', '^', '#', '_', '!', '\\', '|', '~', '`', '\0'}; 40 | 41 | enum {SHX_STATE_1 = 1, SHX_STATE_2}; 42 | 43 | byte to_match_repeats_earlier = 1; 44 | byte to_match_repeats_within = 1; 45 | #define USE_64K_LOOKUP 0 46 | #if USE_64K_LOOKUP == 1 47 | byte lookup[65536]; 48 | #endif 49 | #define NICE_LEN_FOR_PRIOR 7 50 | #define NICE_LEN_FOR_OTHER 12 51 | 52 | unsigned int mask[] = {0x8000, 0xC000, 0xE000, 0xF000, 0xF800, 0xFC00, 0xFE00, 0xFF00}; 53 | int append_bits(char *out, int ol, unsigned int code, int clen, byte state) { 54 | 55 | byte cur_bit; 56 | byte blen; 57 | unsigned char a_byte; 58 | 59 | if (state == SHX_STATE_2) { 60 | // remove change state prefix 61 | if ((code >> 9) == 0x1C) { 62 | code <<= 7; 63 | clen -= 7; 64 | } 65 | //if (code == 14272 && clen == 10) { 66 | // code = 9084; 67 | // clen = 14; 68 | //} 69 | } 70 | while (clen > 0) { 71 | cur_bit = ol % 8; 72 | blen = (clen > 8 ? 8 : clen); 73 | a_byte = (code & mask[blen - 1]) >> 8; 74 | a_byte >>= cur_bit; 75 | if (blen + cur_bit > 8) 76 | blen = (8 - cur_bit); 77 | if (cur_bit == 0) 78 | out[ol / 8] = a_byte; 79 | else 80 | out[ol / 8] |= a_byte; 81 | code <<= blen; 82 | ol += blen; 83 | clen -= blen; 84 | } 85 | return ol; 86 | } 87 | 88 | int encodeCount(char *out, int ol, int count) { 89 | const byte codes[7] = {0x01, 0x82, 0xC3, 0xE5, 0xED, 0xF5, 0xFD}; 90 | const byte bit_len[7] = {2, 5, 7, 9, 12, 16, 17}; 91 | const uint16_t adder[7] = {0, 4, 36, 164, 676, 4772, 0}; 92 | int till = 0; 93 | for (int i = 0; i < 6; i++) { 94 | till += (1 << bit_len[i]); 95 | if (count < till) { 96 | ol = append_bits(out, ol, (codes[i] & 0xF8) << 8, codes[i] & 0x07, 1); 97 | ol = append_bits(out, ol, (count - adder[i]) << (16 - bit_len[i]), bit_len[i], 1); 98 | return ol; 99 | } 100 | } 101 | return ol; 102 | } 103 | 104 | int matchOccurance(const char *in, int len, int l, char *out, int *ol) { 105 | int j, k; 106 | for (j = 0; j < l; j++) { 107 | for (k = j; k < l && (l + k - j) < len; k++) { 108 | if (in[k] != in[l + k - j]) 109 | break; 110 | } 111 | if ((k - j) > (NICE_LEN_FOR_PRIOR - 1)) { 112 | *ol = append_bits(out, *ol, 14144, 10, 1); 113 | *ol = encodeCount(out, *ol, k - j - NICE_LEN_FOR_PRIOR); // len 114 | *ol = encodeCount(out, *ol, l - j - NICE_LEN_FOR_PRIOR + 1); // dist 115 | l += (k - j); 116 | l--; 117 | return l; 118 | } 119 | } 120 | return -l; 121 | } 122 | 123 | int matchLine(const char *in, int len, int l, char *out, int *ol, struct lnk_lst *prev_lines) { 124 | int last_ol = *ol; 125 | int last_len = 0; 126 | int last_dist = 0; 127 | int last_ctx = 0; 128 | int line_ctr = 0; 129 | do { 130 | int i, j, k; 131 | int line_len = strlen(prev_lines->data); 132 | for (j = 0; j < line_len; j++) { 133 | for (i = l, k = j; k < line_len && i < len; k++, i++) { 134 | if (prev_lines->data[k] != in[i]) 135 | break; 136 | } 137 | if ((k - j) >= NICE_LEN_FOR_OTHER) { 138 | if (last_len) { 139 | if (j > last_dist) 140 | continue; 141 | //int saving = ((k - j) - last_len) + (last_dist - j) + (last_ctx - line_ctr); 142 | //if (saving < 0) { 143 | // //printf("No savng: %d\n", saving); 144 | // continue; 145 | //} 146 | *ol = last_ol; 147 | } 148 | last_len = (k - j); 149 | last_dist = j; 150 | last_ctx = line_ctr; 151 | *ol = append_bits(out, *ol, 14080, 10, 1); 152 | *ol = encodeCount(out, *ol, last_len - NICE_LEN_FOR_OTHER); 153 | *ol = encodeCount(out, *ol, last_dist); 154 | *ol = encodeCount(out, *ol, last_ctx); 155 | /* 156 | if ((*ol - last_ol) > (last_len * 4)) { 157 | last_len = 0; 158 | *ol = last_ol; 159 | }*/ 160 | //printf("Len: %d, Dist: %d, Line: %d\n", last_len, last_dist, last_ctx); 161 | } 162 | } 163 | line_ctr++; 164 | prev_lines = prev_lines->previous; 165 | } while (prev_lines && prev_lines->data != NULL); 166 | if (last_len) { 167 | l += last_len; 168 | l--; 169 | return l; 170 | } 171 | return -l; 172 | } 173 | 174 | int shox96_0_2_compress(const char *in, int len, char *out, struct lnk_lst *prev_lines) { 175 | 176 | char *ptr; 177 | byte bits; 178 | byte state; 179 | 180 | int l, ll, ol; 181 | char c_in, c_next, c_prev; 182 | byte is_upper, is_all_upper; 183 | 184 | ol = 0; 185 | c_prev = 0; 186 | #if USE_64K_LOOKUP == 1 187 | memset(lookup, 0, sizeof(lookup)); 188 | #endif 189 | state = SHX_STATE_1; 190 | is_all_upper = 0; 191 | for (l=0; l 0) { 216 | c_prev = in[l - 1]; 217 | continue; 218 | } 219 | l = -l; 220 | #if USE_64K_LOOKUP == 1 221 | } else 222 | lookup[to_lookup] = 1; 223 | #endif 224 | } 225 | if (l < (len - NICE_LEN_FOR_OTHER) && to_match_repeats_earlier) { 226 | if (prev_lines != NULL) { 227 | l = matchLine(in, len, l, out, &ol, prev_lines); 228 | if (l > 0) { 229 | c_prev = in[l - 1]; 230 | continue; 231 | } 232 | l = -l; 233 | } 234 | } 235 | if (state == SHX_STATE_2) { 236 | if (c_in == ' ' && len - 1 > l) 237 | ptr = (char *) memchr(SET2_STR, in[l+1], 42); 238 | else 239 | ptr = (char *) memchr(SET2_STR, c_in, 42); 240 | if (ptr == NULL) { 241 | state = SHX_STATE_1; 242 | ol = append_bits(out, ol, 8192, 4, 1); 243 | } 244 | } 245 | is_upper = 0; 246 | if (c_in >= 'A' && c_in <= 'Z') 247 | is_upper = 1; 248 | else { 249 | if (is_all_upper) { 250 | is_all_upper = 0; 251 | ol = append_bits(out, ol, 8192, 4, state); 252 | } 253 | } 254 | if (is_upper && !is_all_upper) { 255 | for (ll=l+5; ll>=l && ll= 'a' && in[ll] <= 'z') 257 | break; 258 | } 259 | if (ll == l-1) { 260 | ol = append_bits(out, ol, 8704, 8, state); 261 | is_all_upper = 1; 262 | } 263 | } 264 | if (state == SHX_STATE_1 && c_in >= '0' && c_in <= '9') { 265 | ol = append_bits(out, ol, 14336, 7, state); 266 | state = SHX_STATE_2; 267 | } 268 | c_next = 0; 269 | if (l+1 < len) 270 | c_next = in[l+1]; 271 | 272 | c_prev = c_in; 273 | if (c_in >= 32 && c_in <= 126) { 274 | c_in -= 32; 275 | if (is_all_upper && is_upper) 276 | c_in += 32; 277 | if (c_in == 0 && state == SHX_STATE_2) 278 | ol = append_bits(out, ol, 15232, 11, state); 279 | else 280 | ol = append_bits(out, ol, c_95[c_in], l_95[c_in], state); 281 | } else 282 | if (c_in == 13 && c_next == 10) { 283 | ol = append_bits(out, ol, 13824, 9, state); 284 | l++; 285 | c_prev = 10; 286 | } else 287 | if (c_in == 10) { 288 | ol = append_bits(out, ol, 13952, 9, state); 289 | } else 290 | if (c_in == 13) { 291 | ol = append_bits(out, ol, 9064, 13, state); 292 | } else 293 | if (c_in == '\t') { 294 | ol = append_bits(out, ol, 9216, 7, state); 295 | } 296 | } 297 | bits = ol % 8; 298 | if (bits) { 299 | ol = append_bits(out, ol, 14272, 8 - bits, 1); 300 | } 301 | //printf("\n%ld\n", ol); 302 | return ol/8+(ol%8?1:0); 303 | 304 | } 305 | 306 | // Decoder is designed for using less memory, not speed 307 | // Decode lookup table for code index and length 308 | // First 2 bits 00, Next 3 bits indicate index of code from 0, 309 | // last 3 bits indicate code length in bits 310 | // 0, 1, 2, 3, 4, 311 | char vcode[32] = {2 + (0 << 3), 3 + (3 << 3), 3 + (1 << 3), 4 + (6 << 3), 0, 312 | // 5, 6, 7, 8, 9, 10 313 | 4 + (4 << 3), 3 + (2 << 3), 4 + (8 << 3), 0, 0, 0, 314 | // 11, 12, 13, 14, 15 315 | 4 + (7 << 3), 0, 4 + (5 << 3), 0, 5 + (9 << 3), 316 | // 16, 17, 18, 19, 20, 21, 22, 23 317 | 0, 0, 0, 0, 0, 0, 0, 0, 318 | // 24, 25, 26, 27, 28, 29, 30, 31 319 | 0, 0, 0, 0, 0, 0, 0, 5 + (10 << 3)}; 320 | // 0, 1, 2, 3, 4, 5, 6, 7, 321 | char hcode[32] = {1 + (1 << 3), 2 + (0 << 3), 0, 3 + (2 << 3), 0, 0, 0, 5 + (3 << 3), 322 | // 8, 9, 10, 11, 12, 13, 14, 15, 323 | 0, 0, 0, 0, 0, 0, 0, 5 + (5 << 3), 324 | // 16, 17, 18, 19, 20, 21, 22, 23 325 | 0, 0, 0, 0, 0, 0, 0, 5 + (4 << 3), 326 | // 24, 25, 26, 27, 28, 29, 30, 31 327 | 0, 0, 0, 0, 0, 0, 0, 5 + (6 << 3)}; 328 | 329 | enum {SHX_SET1 = 0, SHX_SET1A, SHX_SET1B, SHX_SET2, SHX_SET3, SHX_SET4, SHX_SET4A}; 330 | char sets[][11] = {{' ', ' ', 'e', 't', 'a', 'o', 'i', 'n', 's', 'r', 'l'}, 331 | {'c', 'd', 'h', 'u', 'p', 'm', 'b', 'g', 'w', 'f', 'y'}, 332 | {'v', 'k', 'q', 'j', 'x', 'z', ' ', ' ', ' ', ' ', ' '}, 333 | {' ', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8'}, 334 | {'.', ',', '-', '/', '=', '+', ' ', '(', ')', '$', '%'}, 335 | {'&', ';', ':', '<', '>', '*', '"', '{', '}', '[', ']'}, 336 | {'@', '?', '\'', '^', '#', '_', '!', '\\', '|', '~', '`'}}; 337 | 338 | int getBitVal(const char *in, int bit_no, int count) { 339 | return (in[bit_no >> 3] & (0x80 >> (bit_no % 8)) ? 1 << count : 0); 340 | } 341 | 342 | int getCodeIdx(char *code_type, const char *in, int len, int *bit_no_p) { 343 | int code = 0; 344 | int count = 0; 345 | do { 346 | if (*bit_no_p >= len) 347 | return 199; 348 | code += getBitVal(in, *bit_no_p, count); 349 | (*bit_no_p)++; 350 | count++; 351 | if (code_type[code] && 352 | (code_type[code] & 0x07) == count) { 353 | return code_type[code] >> 3; 354 | } 355 | } while (count < 5); 356 | return 1; // skip if code not found 357 | } 358 | 359 | int getNumFromBits(const char *in, int bit_no, int count) { 360 | int ret = 0; 361 | while (count--) { 362 | ret += getBitVal(in, bit_no++, count); 363 | } 364 | return ret; 365 | } 366 | 367 | int readCount(const char *in, int *bit_no_p, int len) { 368 | const byte bit_len[7] = {5, 2, 7, 9, 12, 16, 17}; 369 | const uint16_t adder[7] = {4, 0, 36, 164, 676, 4772, 0}; 370 | int idx = getCodeIdx(hcode, in, len, bit_no_p); 371 | if (idx > 6) 372 | return 0; 373 | int count = getNumFromBits(in, *bit_no_p, bit_len[idx]) + adder[idx]; 374 | (*bit_no_p) += bit_len[idx]; 375 | return count; 376 | } 377 | 378 | int shox96_0_2_decompress(const char *in, int len, char *out, struct lnk_lst *prev_lines) { 379 | 380 | int dstate; 381 | int bit_no; 382 | byte is_all_upper; 383 | int ol = 0; 384 | bit_no = 0; 385 | dstate = SHX_SET1; 386 | is_all_upper = 0; 387 | 388 | len <<= 3; 389 | out[ol] = 0; 390 | while (bit_no < len) { 391 | int h, v; 392 | char c; 393 | byte is_upper = is_all_upper; 394 | int orig_bit_no = bit_no; 395 | v = getCodeIdx(vcode, in, len, &bit_no); 396 | if (v == 199) { 397 | bit_no = orig_bit_no; 398 | break; 399 | } 400 | h = dstate; 401 | if (v == 0) { 402 | h = getCodeIdx(hcode, in, len, &bit_no); 403 | if (h == 199) { 404 | bit_no = orig_bit_no; 405 | break; 406 | } 407 | if (h == SHX_SET1) { 408 | if (dstate == SHX_SET1) { 409 | if (is_all_upper) { 410 | is_upper = is_all_upper = 0; 411 | continue; 412 | } 413 | v = getCodeIdx(vcode, in, len, &bit_no); 414 | if (v == 199) { 415 | bit_no = orig_bit_no; 416 | break; 417 | } 418 | if (v == 0) { 419 | h = getCodeIdx(hcode, in, len, &bit_no); 420 | if (h == 199) { 421 | bit_no = orig_bit_no; 422 | break; 423 | } 424 | if (h == SHX_SET1) { 425 | is_all_upper = 1; 426 | continue; 427 | } 428 | } 429 | is_upper = 1; 430 | } else { 431 | dstate = SHX_SET1; 432 | continue; 433 | } 434 | } else 435 | if (h == SHX_SET2) { 436 | if (dstate == SHX_SET1) 437 | dstate = SHX_SET2; 438 | continue; 439 | } 440 | if (h != SHX_SET1) { 441 | v = getCodeIdx(vcode, in, len, &bit_no); 442 | if (v == 199) { 443 | bit_no = orig_bit_no; 444 | break; 445 | } 446 | } 447 | } 448 | if (h < 64 && v < 32) 449 | c = sets[h][v]; 450 | if (c >= 'a' && c <= 'z') { 451 | if (is_upper) 452 | c -= 32; 453 | } else { 454 | if (is_upper && dstate == SHX_SET1 && v == 1) 455 | c = '\t'; 456 | if (h == SHX_SET1B) { 457 | switch (v) { 458 | case 6: 459 | out[ol++] = '\r'; 460 | c = '\n'; 461 | break; 462 | case 7: 463 | c = is_upper ? '\r' : '\n'; 464 | break; 465 | case 8: 466 | if (getBitVal(in, bit_no++, 0)) { 467 | int dict_len = readCount(in, &bit_no, len) + NICE_LEN_FOR_PRIOR; 468 | int dist = readCount(in, &bit_no, len) + NICE_LEN_FOR_PRIOR - 1; 469 | memcpy(out + ol, out + ol - dist, dict_len); 470 | ol += dict_len; 471 | } else { 472 | int dict_len = readCount(in, &bit_no, len) + NICE_LEN_FOR_OTHER; 473 | int dist = readCount(in, &bit_no, len); 474 | int ctx = readCount(in, &bit_no, len); 475 | struct lnk_lst *cur_line = prev_lines; 476 | while (ctx--) 477 | cur_line = cur_line->previous; 478 | memmove(out + ol, cur_line->data + dist, dict_len); 479 | ol += dict_len; 480 | } 481 | continue; 482 | case 9: { 483 | int count = readCount(in, &bit_no, len); 484 | count += 4; 485 | char rpt_c = out[ol - 1]; 486 | while (count--) 487 | out[ol++] = rpt_c; 488 | continue; 489 | } 490 | case 10: 491 | continue; 492 | } 493 | } 494 | } 495 | out[ol++] = c; 496 | } 497 | 498 | return ol; 499 | 500 | } 501 | #pragma GCC diagnostic pop 502 | -------------------------------------------------------------------------------- /src/shox96_0_2.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2019 Siara Logics (cc) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * @author Arundale R. 17 | * 18 | */ 19 | #ifndef shox96_0_2 20 | #define shox96_0_2 21 | 22 | struct lnk_lst { 23 | char *data; 24 | struct lnk_lst *previous; 25 | }; 26 | 27 | extern int shox96_0_2_compress(const char *in, int len, char *out, struct lnk_lst *prev_lines); 28 | extern int shox96_0_2_decompress(const char *in, int len, char *out, struct lnk_lst *prev_lines); 29 | #endif 30 | 31 | -------------------------------------------------------------------------------- /src/unishox1.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2019 Siara Logics (cc) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * @author Arundale R. 17 | * 18 | */ 19 | 20 | #pragma GCC diagnostic push 21 | #pragma GCC diagnostic ignored "-Wunused-value" 22 | #pragma GCC diagnostic ignored "-Wunused-label" 23 | #pragma GCC diagnostic ignored "-Wchar-subscripts" 24 | #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" 25 | 26 | #define UNISHOX_VERSION "1.0" 27 | 28 | #ifdef _MSC_VER 29 | #include 30 | #else 31 | #include 32 | #endif 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | #include "unishox1.h" 41 | 42 | typedef unsigned char byte; 43 | 44 | enum {SHX_SET1 = 0, SHX_SET1A, SHX_SET1B, SHX_SET2, SHX_SET3, SHX_SET4, SHX_SET4A}; 45 | char us_vcodes[] = {0, 2, 3, 4, 10, 11, 12, 13, 14, 30, 31}; 46 | char us_vcode_lens[] = {2, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5}; 47 | char us_sets[][11] = {{ 0, ' ', 'e', 0, 't', 'a', 'o', 'i', 'n', 's', 'r'}, 48 | { 0, 'l', 'c', 'd', 'h', 'u', 'p', 'm', 'b', 'g', 'w'}, 49 | {'f', 'y', 'v', 'k', 'q', 'j', 'x', 'z', 0, 0, 0}, 50 | { 0, '9', '0', '1', '2', '3', '4', '5', '6', '7', '8'}, 51 | {'.', ',', '-', '/', '=', '+', ' ', '(', ')', '$', '%'}, 52 | {'&', ';', ':', '<', '>', '*', '"', '{', '}', '[', ']'}, 53 | {'@', '?', '\'', '^', '#', '_', '!', '\\', '|', '~', '`'}}; 54 | 55 | unsigned int c_95[95]; // = {16384, 16256, 15744, 16192, 15328, 15344, 15360, 16064, 15264, 15296, 15712, 15200, 14976, 15040, 14848, 15104, 14528, 14592, 14656, 14688, 14720, 14752, 14784, 14816, 14832, 14464, 15552, 15488, 15616, 15168, 15680, 16000, 15872, 10752, 8576, 8192, 8320, 9728, 8672, 8608, 8384, 11264, 9024, 8992, 12160, 8544, 11520, 11008, 8512, 9008, 12032, 11776, 10240, 8448, 8960, 8640, 9040, 8688, 9048, 15840, 16288, 15856, 16128, 16224, 16368, 40960, 6144, 0, 2048, 24576, 7680, 6656, 3072, 49152, 13312, 12800, 63488, 5632, 53248, 45056, 5120, 13056, 61440, 57344, 32768, 4096, 12288, 7168, 13568, 7936, 13696, 15776, 16320, 15808, 16352}; 56 | unsigned char l_95[95]; // = { 3, 11, 11, 11, 12, 12, 9, 10, 11, 11, 11, 11, 10, 10, 9, 10, 10, 10, 11, 11, 11, 11, 11, 12, 12, 10, 10, 10, 10, 11, 11, 10, 9, 8, 11, 9, 10, 7, 12, 11, 10, 8, 12, 12, 9, 11, 8, 8, 11, 12, 9, 8, 7, 10, 11, 11, 13, 12, 13, 12, 11, 12, 10, 11, 12, 4, 7, 5, 6, 3, 8, 7, 6, 4, 8, 8, 5, 7, 4, 4, 7, 8, 5, 4, 3, 6, 7, 7, 9, 8, 9, 11, 11, 11, 12}; 57 | //unsigned char c[] = { ' ', '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~'}; 58 | 59 | const int UTF8_MASK[] = {0xE0, 0xF0, 0xF8}; 60 | const int UTF8_PREFIX[] = {0xC0, 0xE0, 0xF0}; 61 | 62 | enum {SHX_STATE_1 = 1, SHX_STATE_2, SHX_STATE_UNI}; 63 | 64 | byte to_match_repeats = 1; 65 | #define USE_64K_LOOKUP 0 66 | #if USE_64K_LOOKUP == 1 67 | byte lookup[65536]; 68 | #endif 69 | #define NICE_LEN 5 70 | 71 | #define TERM_CODE 0x37C0 72 | #define TERM_CODE_LEN 10 73 | #define DICT_CODE 0x0000 74 | #define DICT_CODE_LEN 5 75 | #define DICT_OTHER_CODE 0x0000 // not used 76 | #define DICT_OTHER_CODE_LEN 6 77 | #define RPT_CODE 0x2370 78 | #define RPT_CODE_LEN 13 79 | #define BACK2_STATE1_CODE 0x2000 80 | #define BACK2_STATE1_CODE_LEN 4 81 | #define BACK_FROM_UNI_CODE 0xFE00 82 | #define BACK_FROM_UNI_CODE_LEN 8 83 | #define CRLF_CODE 0x3780 84 | #define CRLF_CODE_LEN 10 85 | #define LF_CODE 0x3700 86 | #define LF_CODE_LEN 9 87 | #define TAB_CODE 0x2400 88 | #define TAB_CODE_LEN 7 89 | #define UNI_CODE 0x8000 90 | #define UNI_CODE_LEN 3 91 | #define UNI_STATE_SPL_CODE 0xF800 92 | #define UNI_STATE_SPL_CODE_LEN 5 93 | #define UNI_STATE_DICT_CODE 0xFC00 94 | #define UNI_STATE_DICT_CODE_LEN 7 95 | #define CONT_UNI_CODE 0x2800 96 | #define CONT_UNI_CODE_LEN 7 97 | #define ALL_UPPER_CODE 0x2200 98 | #define ALL_UPPER_CODE_LEN 8 99 | #define SW2_STATE2_CODE 0x3800 100 | #define SW2_STATE2_CODE_LEN 7 101 | #define ST2_SPC_CODE 0x3B80 102 | #define ST2_SPC_CODE_LEN 11 103 | #define BIN_CODE 0x2000 104 | #define BIN_CODE_LEN 9 105 | 106 | //void checkPreus_vcodes(char c, int prev_code, char prev_code_len, int c_95, char l_95) { 107 | // if (prev_code != c_95 || prev_code_len != l_95) { 108 | // printf("Code mismatch: %d: %d!=%d, %d!=%d\n", c, prev_code, c_95, prev_code_len, l_95); 109 | // } 110 | //} 111 | 112 | byte is_inited = 0; 113 | void init_coder() { 114 | if (is_inited) 115 | return; 116 | for (int i = 0; i < 7; i++) { 117 | for (int j = 0; j < 11; j++) { 118 | char c = us_sets[i][j]; 119 | if (c != 0 && c != 32) { 120 | int ascii = c - 32; 121 | //int prev_code = c_95[ascii]; 122 | //int prev_code_len = l_95[ascii]; 123 | switch (i) { 124 | case SHX_SET1: // just us_vcode 125 | c_95[ascii] = (us_vcodes[j] << (16 - us_vcode_lens[j])); 126 | l_95[ascii] = us_vcode_lens[j]; 127 | //checkPreus_vcodes(c, prev_code, prev_code_len, c_95[ascii], l_95[ascii]); 128 | if (c >= 'a' && c <= 'z') { 129 | ascii -= ('a' - 'A'); 130 | //prev_code = c_95[ascii]; 131 | //prev_code_len = l_95[ascii]; 132 | c_95[ascii] = (2 << 12) + (us_vcodes[j] << (12 - us_vcode_lens[j])); 133 | l_95[ascii] = 4 + us_vcode_lens[j]; 134 | } 135 | break; 136 | case SHX_SET1A: // 000 + us_vcode 137 | c_95[ascii] = 0 + (us_vcodes[j] << (13 - us_vcode_lens[j])); 138 | l_95[ascii] = 3 + us_vcode_lens[j]; 139 | //checkPreus_vcodes(c, prev_code, prev_code_len, c_95[ascii], l_95[ascii]); 140 | if (c >= 'a' && c <= 'z') { 141 | ascii -= ('a' - 'A'); 142 | //prev_code = c_95[ascii]; 143 | //prev_code_len = l_95[ascii]; 144 | c_95[ascii] = (2 << 12) + 0 + (us_vcodes[j] << (9 - us_vcode_lens[j])); 145 | l_95[ascii] = 4 + 3 + us_vcode_lens[j]; 146 | } 147 | break; 148 | case SHX_SET1B: // 00110 + us_vcode 149 | c_95[ascii] = (6 << 11) + (us_vcodes[j] << (11 - us_vcode_lens[j])); 150 | l_95[ascii] = 5 + us_vcode_lens[j]; 151 | //checkPreus_vcodes(c, prev_code, prev_code_len, c_95[ascii], l_95[ascii]); 152 | if (c >= 'a' && c <= 'z') { 153 | ascii -= ('a' - 'A'); 154 | //prev_code = c_95[ascii]; 155 | //prev_code_len = l_95[ascii]; 156 | c_95[ascii] = (2 << 12) + (6 << 7) + (us_vcodes[j] << (7 - us_vcode_lens[j])); 157 | l_95[ascii] = 4 + 5 + us_vcode_lens[j]; 158 | } 159 | break; 160 | case SHX_SET2: // 0011100 + us_vcode 161 | c_95[ascii] = (28 << 9) + (us_vcodes[j] << (9 - us_vcode_lens[j])); 162 | l_95[ascii] = 7 + us_vcode_lens[j]; 163 | break; 164 | case SHX_SET3: // 0011101 + us_vcode 165 | c_95[ascii] = (29 << 9) + (us_vcodes[j] << (9 - us_vcode_lens[j])); 166 | l_95[ascii] = 7 + us_vcode_lens[j]; 167 | break; 168 | case SHX_SET4: // 0011110 + us_vcode 169 | c_95[ascii] = (30 << 9) + (us_vcodes[j] << (9 - us_vcode_lens[j])); 170 | l_95[ascii] = 7 + us_vcode_lens[j]; 171 | break; 172 | case SHX_SET4A: // 0011111 + us_vcode 173 | c_95[ascii] = (31 << 9) + (us_vcodes[j] << (9 - us_vcode_lens[j])); 174 | l_95[ascii] = 7 + us_vcode_lens[j]; 175 | } 176 | //checkPreus_vcodes(c, prev_code, prev_code_len, c_95[ascii], l_95[ascii]); 177 | } 178 | } 179 | } 180 | c_95[0] = 16384; 181 | l_95[0] = 3; 182 | 183 | is_inited = 1; 184 | } 185 | 186 | unsigned int us_mask[] = {0x8000, 0xC000, 0xE000, 0xF000, 0xF800, 0xFC00, 0xFE00, 0xFF00}; 187 | int append_bits(char *out, int ol, unsigned int code, int clen, byte state) { 188 | 189 | byte cur_bit; 190 | byte blen; 191 | unsigned char a_byte; 192 | 193 | //printf("%d,%x,%d,%d\n", ol, code, clen, state); 194 | 195 | if (state == SHX_STATE_2) { 196 | // remove change state prefix 197 | if ((code >> 9) == 0x1C) { 198 | code <<= 7; 199 | clen -= 7; 200 | } 201 | //if (code == 14272 && clen == 10) { 202 | // code = 9084; 203 | // clen = 14; 204 | //} 205 | } 206 | while (clen > 0) { 207 | cur_bit = ol % 8; 208 | blen = (clen > 8 ? 8 : clen); 209 | a_byte = (code & us_mask[blen - 1]) >> 8; 210 | a_byte >>= cur_bit; 211 | if (blen + cur_bit > 8) 212 | blen = (8 - cur_bit); 213 | if (cur_bit == 0) 214 | out[ol / 8] = a_byte; 215 | else 216 | out[ol / 8] |= a_byte; 217 | code <<= blen; 218 | ol += blen; 219 | clen -= blen; 220 | } 221 | return ol; 222 | } 223 | 224 | int encodeCount(char *out, int ol, int count) { 225 | // First five bits are code and Last three bits of codes represent length 226 | const byte codes[7] = {0x01, 0x82, 0xC3, 0xE5, 0xED, 0xF5, 0xFD}; 227 | const byte bit_len[7] = {2, 5, 7, 9, 12, 16, 17}; 228 | const uint16_t adder[7] = {0, 4, 36, 164, 676, 4772, 0}; 229 | int till = 0; 230 | for (int i = 0; i < 6; i++) { 231 | till += (1 << bit_len[i]); 232 | if (count < till) { 233 | ol = append_bits(out, ol, (codes[i] & 0xF8) << 8, codes[i] & 0x07, 1); 234 | ol = append_bits(out, ol, (count - adder[i]) << (16 - bit_len[i]), bit_len[i], 1); 235 | return ol; 236 | } 237 | } 238 | return ol; 239 | } 240 | 241 | //const byte uni_bit_len[4] = {5, 12, 15, 23}; 242 | //const int32_t uni_adder[4] = {0, 32, 4128, 36896}; 243 | //const byte uni_bit_len[4] = {6, 12, 15, 23}; 244 | //const int32_t uni_adder[4] = {0, 64, 4160, 36928}; 245 | //const byte uni_bit_len[4] = {6, 12, 14, 21}; 246 | //const int32_t uni_adder[4] = {0, 64, 4160, 20544}; 247 | //const byte uni_bit_len[7] = {0, 3, 6, 12, 14, 16, 21}; 248 | //const int32_t uni_adder[7] = {0, 1, 9, 73, 4169, 20553, 86089}; 249 | const byte uni_bit_len[5] = {6, 12, 14, 16, 21}; 250 | const int32_t uni_adder[5] = {0, 64, 4160, 20544, 86080}; 251 | 252 | int encodeUnicode(char *out, int ol, int32_t code, int32_t prev_code) { 253 | uint16_t spl_code = (code == ',' ? 0xE000 : 254 | ((code == '.' || code == 0x3002) ? 0xE800 : (code == ' ' ? 0 : 255 | (code == 13 ? 0xF000 : (code == 10 ? 0xF800 : 0xFFFF))))); 256 | if (spl_code != 0xFFFF) { 257 | uint16_t spl_code_len = (code == ',' ? 5 : 258 | ((code == '.' || code == 0x3002) ? 5 : (code == ' ' ? 1 : 259 | (code == 13 ? 5 : (code == 10 ? 5 : 0xFFFF))))); 260 | ol = append_bits(out, ol, UNI_STATE_SPL_CODE, UNI_STATE_SPL_CODE_LEN, SHX_STATE_UNI); 261 | ol = append_bits(out, ol, spl_code, spl_code_len, 1); 262 | return ol; 263 | } 264 | // First five bits are code and Last three bits of codes represent length 265 | //const byte codes[8] = {0x00, 0x42, 0x83, 0xA3, 0xC3, 0xE4, 0xF5, 0xFD}; 266 | const byte codes[6] = {0x01, 0x82, 0xC3, 0xE4, 0xF5, 0xFD}; 267 | int32_t till = 0; 268 | int orig_ol = ol; 269 | for (int i = 0; i < 5; i++) { 270 | till += (1 << uni_bit_len[i]); 271 | int32_t diff = abs(code - prev_code); 272 | if (diff < till) { 273 | ol = append_bits(out, ol, (codes[i] & 0xF8) << 8, codes[i] & 0x07, 1); 274 | //if (diff) { 275 | ol = append_bits(out, ol, prev_code > code ? 0x8000 : 0, 1, 1); 276 | if (uni_bit_len[i] > 16) { 277 | int32_t val = diff - uni_adder[i]; 278 | int excess_bits = uni_bit_len[i] - 16; 279 | ol = append_bits(out, ol, val >> excess_bits, 16, 1); 280 | ol = append_bits(out, ol, (val & ((1 << excess_bits) - 1)) << (16 - excess_bits), excess_bits, 1); 281 | } else 282 | ol = append_bits(out, ol, (diff - uni_adder[i]) << (16 - uni_bit_len[i]), uni_bit_len[i], 1); 283 | //} 284 | //printf("bits:%d\n", ol-orig_ol); 285 | return ol; 286 | } 287 | } 288 | return ol; 289 | } 290 | 291 | int readUTF8(const char *in, int len, int l, int *utf8len) { 292 | int bc = 0; 293 | int uni = 0; 294 | byte c_in = in[l]; 295 | for (; bc < 3; bc++) { 296 | if (UTF8_PREFIX[bc] == (c_in & UTF8_MASK[bc]) && len - (bc + 1) > l) { 297 | int j = 0; 298 | uni = c_in & ~UTF8_MASK[bc] & 0xFF; 299 | do { 300 | uni <<= 6; 301 | uni += (in[l + j + 1] & 0x3F); 302 | } while (j++ < bc); 303 | break; 304 | } 305 | } 306 | if (bc < 3) { 307 | *utf8len = bc + 1; 308 | return uni; 309 | } 310 | return 0; 311 | } 312 | 313 | int matchOccurance(const char *in, int len, int l, char *out, int *ol, byte *state, byte *is_all_upper) { 314 | int j, k; 315 | int longest_dist = 0; 316 | int longest_len = 0; 317 | for (j = l - NICE_LEN; j >= 0; j--) { 318 | for (k = l; k < len && j + k - l < l; k++) { 319 | if (in[k] != in[j + k - l]) 320 | break; 321 | } 322 | while ((((unsigned char) in[k]) >> 6) == 2) 323 | k--; // Skip partial UTF-8 matches 324 | //if ((in[k - 1] >> 3) == 0x1E || (in[k - 1] >> 4) == 0x0E || (in[k - 1] >> 5) == 0x06) 325 | // k--; 326 | if (k - l > NICE_LEN - 1) { 327 | int match_len = k - l - NICE_LEN; 328 | int match_dist = l - j - NICE_LEN + 1; 329 | if (match_len > longest_len) { 330 | longest_len = match_len; 331 | longest_dist = match_dist; 332 | } 333 | } 334 | } 335 | if (longest_len) { 336 | if (*state == SHX_STATE_2 || *is_all_upper) { 337 | *is_all_upper = 0; 338 | *state = SHX_STATE_1; 339 | *ol = append_bits(out, *ol, BACK2_STATE1_CODE, BACK2_STATE1_CODE_LEN, *state); 340 | } 341 | if (*state == SHX_STATE_UNI) 342 | *ol = append_bits(out, *ol, UNI_STATE_DICT_CODE, UNI_STATE_DICT_CODE_LEN, SHX_STATE_UNI); 343 | else 344 | *ol = append_bits(out, *ol, DICT_CODE, DICT_CODE_LEN, 1); 345 | //printf("Len:%d / Dist:%d\n", longest_len, longest_dist); 346 | *ol = encodeCount(out, *ol, longest_len); 347 | *ol = encodeCount(out, *ol, longest_dist); 348 | l += (longest_len + NICE_LEN); 349 | l--; 350 | return l; 351 | } 352 | return -l; 353 | } 354 | 355 | int matchLine(const char *in, int len, int l, char *out, int *ol, struct us_lnk_lst *prev_lines, byte *state, byte *is_all_upper) { 356 | int last_ol = *ol; 357 | int last_len = 0; 358 | int last_dist = 0; 359 | int last_ctx = 0; 360 | int line_ctr = 0; 361 | int j = 0; 362 | do { 363 | int i, k; 364 | int line_len = strlen(prev_lines->data); 365 | int limit = (line_ctr == 0 ? l : line_len); 366 | for (; j < limit; j++) { 367 | for (i = l, k = j; k < line_len && i < len; k++, i++) { 368 | if (prev_lines->data[k] != in[i]) 369 | break; 370 | } 371 | while ((((unsigned char) prev_lines->data[k]) >> 6) == 2) 372 | k--; // Skip partial UTF-8 matches 373 | if ((k - j) >= NICE_LEN) { 374 | if (last_len) { 375 | if (j > last_dist) 376 | continue; 377 | //int saving = ((k - j) - last_len) + (last_dist - j) + (last_ctx - line_ctr); 378 | //if (saving < 0) { 379 | // //printf("No savng: %d\n", saving); 380 | // continue; 381 | //} 382 | *ol = last_ol; 383 | } 384 | last_len = (k - j); 385 | last_dist = j; 386 | last_ctx = line_ctr; 387 | if (*state == SHX_STATE_2 || *is_all_upper) { 388 | *is_all_upper = 0; 389 | *state = SHX_STATE_1; 390 | *ol = append_bits(out, *ol, BACK2_STATE1_CODE, BACK2_STATE1_CODE_LEN, *state); 391 | } 392 | if (*state == SHX_STATE_UNI) 393 | *ol = append_bits(out, *ol, UNI_STATE_DICT_CODE, UNI_STATE_DICT_CODE_LEN, SHX_STATE_UNI); 394 | else 395 | *ol = append_bits(out, *ol, DICT_CODE, DICT_CODE_LEN, 1); 396 | *ol = encodeCount(out, *ol, last_len - NICE_LEN); 397 | *ol = encodeCount(out, *ol, last_dist); 398 | *ol = encodeCount(out, *ol, last_ctx); 399 | /* 400 | if ((*ol - last_ol) > (last_len * 4)) { 401 | last_len = 0; 402 | *ol = last_ol; 403 | }*/ 404 | printf("Len: %d, Dist: %d, Line: %d\n", last_len, last_dist, last_ctx); 405 | j += last_len; 406 | } 407 | } 408 | line_ctr++; 409 | prev_lines = prev_lines->previous; 410 | } while (prev_lines && prev_lines->data != NULL); 411 | if (last_len) { 412 | l += last_len; 413 | l--; 414 | return l; 415 | } 416 | return -l; 417 | } 418 | 419 | int unishox1_compress(const char *in, int len, char *out, struct us_lnk_lst *prev_lines) { 420 | 421 | char *ptr; 422 | byte bits; 423 | byte state; 424 | 425 | int l, ll, ol; 426 | char c_in, c_next; 427 | int prev_uni; 428 | byte is_upper, is_all_upper; 429 | 430 | init_coder(); 431 | ol = 0; 432 | prev_uni = 0; 433 | #if USE_64K_LOOKUP == 1 434 | memset(lookup, 0, sizeof(lookup)); 435 | #endif 436 | state = SHX_STATE_1; 437 | is_all_upper = 0; 438 | for (l=0; l 0) { 465 | continue; 466 | } 467 | l = -l; 468 | } else { 469 | #if USE_64K_LOOKUP == 1 470 | uint16_t to_lookup = c_in ^ in[l + 1] + ((in[l + 2] ^ in[l + 3]) << 8); 471 | if (lookup[to_lookup]) { 472 | #endif 473 | l = matchOccurance(in, len, l, out, &ol, &state, &is_all_upper); 474 | if (l > 0) { 475 | continue; 476 | } 477 | l = -l; 478 | #if USE_64K_LOOKUP == 1 479 | } else 480 | lookup[to_lookup] = 1; 481 | #endif 482 | } 483 | } 484 | 485 | if (state == SHX_STATE_UNI && (c_in == '.' || c_in == ' ' 486 | || c_in == ',' || c_in == 13 || c_in == 10)) { 487 | ol = encodeUnicode(out, ol, c_in, prev_uni); 488 | continue; 489 | } 490 | 491 | if (state == SHX_STATE_2) { 492 | if ((c_in >= ' ' && c_in <= '@') || 493 | (c_in >= '[' && c_in <= '`') || 494 | (c_in >= '{' && c_in <= '~')) { 495 | } else { 496 | state = SHX_STATE_1; 497 | ol = append_bits(out, ol, BACK2_STATE1_CODE, BACK2_STATE1_CODE_LEN, state); 498 | } 499 | } 500 | if (state == SHX_STATE_UNI && c_in >= 0 && c_in <= 127) { 501 | ol = append_bits(out, ol, BACK_FROM_UNI_CODE, BACK_FROM_UNI_CODE_LEN, state); 502 | state = SHX_STATE_1; 503 | } 504 | is_upper = 0; 505 | if (c_in >= 'A' && c_in <= 'Z') 506 | is_upper = 1; 507 | else { 508 | if (is_all_upper) { 509 | is_all_upper = 0; 510 | ol = append_bits(out, ol, BACK2_STATE1_CODE, BACK2_STATE1_CODE_LEN, state); 511 | } 512 | } 513 | c_next = 0; 514 | if (l+1 < len) 515 | c_next = in[l+1]; 516 | 517 | if (c_in >= 32 && c_in <= 126) { 518 | if (is_upper && !is_all_upper) { 519 | for (ll=l+5; ll>=l && ll 'Z') 521 | break; 522 | } 523 | if (ll == l-1) { 524 | ol = append_bits(out, ol, ALL_UPPER_CODE, ALL_UPPER_CODE_LEN, state); 525 | is_all_upper = 1; 526 | } 527 | } 528 | if (state == SHX_STATE_1 && c_in >= '0' && c_in <= '9') { 529 | ol = append_bits(out, ol, SW2_STATE2_CODE, SW2_STATE2_CODE_LEN, state); 530 | state = SHX_STATE_2; 531 | } 532 | c_in -= 32; 533 | if (is_all_upper && is_upper) 534 | c_in += 32; 535 | if (c_in == 0 && state == SHX_STATE_2) 536 | ol = append_bits(out, ol, ST2_SPC_CODE, ST2_SPC_CODE_LEN, state); 537 | else 538 | ol = append_bits(out, ol, c_95[c_in], l_95[c_in], state); 539 | } else 540 | if (c_in == 13 && c_next == 10) { 541 | ol = append_bits(out, ol, CRLF_CODE, CRLF_CODE_LEN, state); 542 | l++; 543 | } else 544 | if (c_in == 10) { 545 | ol = append_bits(out, ol, LF_CODE, LF_CODE_LEN, state); 546 | } else 547 | if (c_in == '\t') { 548 | ol = append_bits(out, ol, TAB_CODE, TAB_CODE_LEN, state); 549 | } else { 550 | int utf8len; 551 | int uni = readUTF8(in, len, l, &utf8len); 552 | if (uni) { 553 | l += utf8len; 554 | if (state != SHX_STATE_UNI) { 555 | int uni2 = readUTF8(in, len, l + 1, &utf8len); 556 | if (uni2) { 557 | state = SHX_STATE_UNI; 558 | ol = append_bits(out, ol, CONT_UNI_CODE, CONT_UNI_CODE_LEN, 1); 559 | } else { 560 | ol = append_bits(out, ol, UNI_CODE, UNI_CODE_LEN, 1); 561 | } 562 | } 563 | ol = encodeUnicode(out, ol, uni, prev_uni); 564 | //printf("%d:%d:%d,", l, utf8len, uni); 565 | if (uni != 0x3002) 566 | prev_uni = uni; 567 | } else { 568 | if (state == SHX_STATE_UNI) { 569 | state = SHX_STATE_1; 570 | ol = append_bits(out, ol, BACK_FROM_UNI_CODE, BACK_FROM_UNI_CODE_LEN, state); 571 | } 572 | printf("Bin:%d:%x\n", (unsigned char) c_in, (unsigned char) c_in); 573 | ol = append_bits(out, ol, BIN_CODE, BIN_CODE_LEN, state); 574 | ol = encodeCount(out, ol, (unsigned char) c_in); 575 | } 576 | } 577 | } 578 | if (state == SHX_STATE_UNI) { 579 | ol = append_bits(out, ol, BACK_FROM_UNI_CODE, BACK_FROM_UNI_CODE_LEN, state); 580 | } 581 | bits = ol % 8; 582 | if (bits) { 583 | ol = append_bits(out, ol, TERM_CODE, 8 - bits, 1); 584 | } 585 | //printf("\n%ld\n", ol); 586 | return ol/8+(ol%8?1:0); 587 | 588 | } 589 | 590 | // Decoder is designed for using less memory, not speed 591 | // Decode lookup table for code index and length 592 | // First 2 bits 00, Next 3 bits indicate index of code from 0, 593 | // last 3 bits indicate code length in bits 594 | // 0, 1, 2, 3, 4, 595 | char us_vcode[32] = {2 + (0 << 3), 3 + (3 << 3), 3 + (1 << 3), 4 + (6 << 3), 0, 596 | // 5, 6, 7, 8, 9, 10 597 | 4 + (4 << 3), 3 + (2 << 3), 4 + (8 << 3), 0, 0, 0, 598 | // 11, 12, 13, 14, 15 599 | 4 + (7 << 3), 0, 4 + (5 << 3), 0, 5 + (9 << 3), 600 | // 16, 17, 18, 19, 20, 21, 22, 23 601 | 0, 0, 0, 0, 0, 0, 0, 0, 602 | // 24, 25, 26, 27, 28, 29, 30, 31 603 | 0, 0, 0, 0, 0, 0, 0, 5 + (10 << 3)}; 604 | // 0, 1, 2, 3, 4, 5, 6, 7, 605 | char us_hcode[32] = {1 + (1 << 3), 2 + (0 << 3), 0, 3 + (2 << 3), 0, 0, 0, 5 + (3 << 3), 606 | // 8, 9, 10, 11, 12, 13, 14, 15, 607 | 0, 0, 0, 0, 0, 0, 0, 5 + (5 << 3), 608 | // 16, 17, 18, 19, 20, 21, 22, 23 609 | 0, 0, 0, 0, 0, 0, 0, 5 + (4 << 3), 610 | // 24, 25, 26, 27, 28, 29, 30, 31 611 | 0, 0, 0, 0, 0, 0, 0, 5 + (6 << 3)}; 612 | 613 | int getBitVal(const char *in, int bit_no, int count) { 614 | return (in[bit_no >> 3] & (0x80 >> (bit_no % 8)) ? 1 << count : 0); 615 | } 616 | 617 | int getCodeIdx(char *code_type, const char *in, int len, int *bit_no_p) { 618 | int code = 0; 619 | int count = 0; 620 | do { 621 | if (*bit_no_p >= len) 622 | return 199; 623 | code += getBitVal(in, *bit_no_p, count); 624 | (*bit_no_p)++; 625 | count++; 626 | if (code_type[code] && 627 | (code_type[code] & 0x07) == count) { 628 | return code_type[code] >> 3; 629 | } 630 | } while (count < 5); 631 | return 1; // skip if code not found 632 | } 633 | 634 | int32_t getNumFromBits(const char *in, int bit_no, int count) { 635 | int32_t ret = 0; 636 | while (count--) { 637 | ret += getBitVal(in, bit_no, count); 638 | bit_no++; 639 | } 640 | return ret; 641 | } 642 | 643 | int readCount(const char *in, int *bit_no_p, int len) { 644 | const byte bit_len[7] = {5, 2, 7, 9, 12, 16, 17}; 645 | const uint16_t adder[7] = {4, 0, 36, 164, 676, 4772, 0}; 646 | int idx = getCodeIdx(us_hcode, in, len, bit_no_p); 647 | if (idx > 6) 648 | return 0; 649 | int count = getNumFromBits(in, *bit_no_p, bit_len[idx]) + adder[idx]; 650 | (*bit_no_p) += bit_len[idx]; 651 | return count; 652 | } 653 | 654 | int32_t readUnicode(const char *in, int *bit_no_p, int len) { 655 | int code = 0; 656 | for (int i = 0; i < 5; i++) { 657 | code += getBitVal(in, *bit_no_p, i); 658 | (*bit_no_p)++; 659 | //int idx = (code == 0 && i == 1 ? 0 : (code == 2 && i == 1 ? 1 : 660 | // (code == 1 && i == 2 ? 2 : (code == 5 && i == 2 ? 3 : 661 | // (code == 3 && i == 2 ? 4 : (code == 7 && i == 3 ? 5 : 662 | // (code == 15 && i == 4 ? 6 : 663 | // (code == 31 && i == 4 ? 7 : -1)))))))); 664 | int idx = (code == 0 && i == 0 ? 0 : (code == 1 && i == 1 ? 1 : 665 | (code == 3 && i == 2 ? 2 : (code == 7 && i == 3 ? 3 : 666 | (code == 15 && i == 4 ? 4 : 667 | (code == 31 && i == 4 ? 5 : -1)))))); 668 | //printf("%d\n", code); 669 | //if (idx == 0) 670 | // return 0; 671 | if (idx == 5) { 672 | int idx = getCodeIdx(us_hcode, in, len, bit_no_p); 673 | return 0x7FFFFF00 + idx; 674 | } 675 | if (idx >= 0) { 676 | int sign = getBitVal(in, *bit_no_p, 1); 677 | (*bit_no_p)++; 678 | int32_t count = getNumFromBits(in, *bit_no_p, uni_bit_len[idx]); 679 | count += uni_adder[idx]; 680 | (*bit_no_p) += uni_bit_len[idx]; 681 | return sign ? -count : count; 682 | } 683 | } 684 | return 0; 685 | } 686 | 687 | void writeUTF8(char *out, int *ol, int uni) { 688 | if (uni < (1 << 11)) { 689 | out[(*ol)++] = (0xC0 + (uni >> 6)); 690 | out[(*ol)++] = (0x80 + (uni & 63)); 691 | } else 692 | if (uni < (1 << 16)) { 693 | out[(*ol)++] = (0xE0 + (uni >> 12)); 694 | out[(*ol)++] = (0x80 + ((uni >> 6) & 63)); 695 | out[(*ol)++] = (0x80 + (uni & 63)); 696 | } else { 697 | out[(*ol)++] = (0xF0 + (uni >> 18)); 698 | out[(*ol)++] = (0x80 + ((uni >> 12) & 63)); 699 | out[(*ol)++] = (0x80 + ((uni >> 6) & 63)); 700 | out[(*ol)++] = (0x80 + (uni & 63)); 701 | } 702 | } 703 | 704 | int decodeRepeat(const char *in, int len, char *out, int ol, int *bit_no, struct us_lnk_lst *prev_lines) { 705 | if (prev_lines) { 706 | int dict_len = readCount(in, bit_no, len) + NICE_LEN; 707 | int dist = readCount(in, bit_no, len); 708 | int ctx = readCount(in, bit_no, len); 709 | struct us_lnk_lst *cur_line = prev_lines; 710 | while (ctx--) 711 | cur_line = cur_line->previous; 712 | memmove(out + ol, cur_line->data + dist, dict_len); 713 | ol += dict_len; 714 | } else { 715 | int dict_len = readCount(in, bit_no, len) + NICE_LEN; 716 | int dist = readCount(in, bit_no, len) + NICE_LEN - 1; 717 | memcpy(out + ol, out + ol - dist, dict_len); 718 | ol += dict_len; 719 | } 720 | return ol; 721 | } 722 | 723 | int unishox1_decompress(const char *in, int len, char *out, struct us_lnk_lst *prev_lines) { 724 | 725 | int dstate; 726 | int bit_no; 727 | byte is_all_upper; 728 | 729 | init_coder(); 730 | int ol = 0; 731 | bit_no = 0; 732 | dstate = SHX_SET1; 733 | is_all_upper = 0; 734 | 735 | int prev_uni = 0; 736 | 737 | len <<= 3; 738 | out[ol] = 0; 739 | while (bit_no < len) { 740 | int h, v; 741 | char c = 0; 742 | byte is_upper = is_all_upper; 743 | int orig_bit_no = bit_no; 744 | v = getCodeIdx(us_vcode, in, len, &bit_no); 745 | if (v == 199) { 746 | bit_no = orig_bit_no; 747 | break; 748 | } 749 | h = dstate; 750 | if (v == 0) { 751 | h = getCodeIdx(us_hcode, in, len, &bit_no); 752 | if (h == 199) { 753 | bit_no = orig_bit_no; 754 | break; 755 | } 756 | if (h == SHX_SET1) { 757 | if (dstate == SHX_SET1) { 758 | if (is_all_upper) { 759 | is_upper = is_all_upper = 0; 760 | continue; 761 | } 762 | v = getCodeIdx(us_vcode, in, len, &bit_no); 763 | if (v == 199) { 764 | bit_no = orig_bit_no; 765 | break; 766 | } 767 | if (v == 0) { 768 | h = getCodeIdx(us_hcode, in, len, &bit_no); 769 | if (h == 199) { 770 | bit_no = orig_bit_no; 771 | break; 772 | } 773 | if (h == SHX_SET1) { 774 | is_all_upper = 1; 775 | continue; 776 | } 777 | } 778 | is_upper = 1; 779 | } else { 780 | dstate = SHX_SET1; 781 | continue; 782 | } 783 | } else 784 | if (h == SHX_SET2) { 785 | if (dstate == SHX_SET1) 786 | dstate = SHX_SET2; 787 | continue; 788 | } 789 | if (h != SHX_SET1) { 790 | v = getCodeIdx(us_vcode, in, len, &bit_no); 791 | if (v == 199) { 792 | bit_no = orig_bit_no; 793 | break; 794 | } 795 | } 796 | } 797 | if (v == 0 && h == SHX_SET1A) { 798 | if (is_upper) { 799 | out[ol++] = readCount(in, &bit_no, len); 800 | } else { 801 | ol = decodeRepeat(in, len, out, ol, &bit_no, prev_lines); 802 | } 803 | continue; 804 | } 805 | if (h == SHX_SET1 && v == 3) { 806 | do { 807 | int32_t delta = readUnicode(in, &bit_no, len); 808 | if ((delta >> 8) == 0x7FFFFF) { 809 | int spl_code_idx = delta & 0x000000FF; 810 | if (spl_code_idx == 2) 811 | break; 812 | switch (spl_code_idx) { 813 | case 1: 814 | out[ol++] = ' '; 815 | break; 816 | case 0: 817 | ol = decodeRepeat(in, len, out, ol, &bit_no, prev_lines); 818 | break; 819 | case 3: 820 | out[ol++] = ','; 821 | break; 822 | case 4: 823 | if (prev_uni > 0x3000) 824 | writeUTF8(out, &ol, 0x3002); 825 | else 826 | out[ol++] = '.'; 827 | break; 828 | case 5: 829 | out[ol++] = 13; 830 | break; 831 | case 6: 832 | out[ol++] = 10; 833 | } 834 | } else { 835 | prev_uni += delta; 836 | writeUTF8(out, &ol, prev_uni); 837 | } 838 | } while (is_upper); 839 | //printf("Sign: %d, bitno: %d\n", sign, bit_no); 840 | //printf("Code: %d\n", prev_uni); 841 | //printf("BitNo: %d\n", bit_no); 842 | continue; 843 | } 844 | if (h < 64 && v < 32) 845 | c = us_sets[h][v]; 846 | if (c >= 'a' && c <= 'z') { 847 | if (is_upper) 848 | c -= 32; 849 | } else { 850 | if (is_upper && dstate == SHX_SET1 && v == 1) 851 | c = '\t'; 852 | if (h == SHX_SET1B) { 853 | switch (v) { 854 | case 9: 855 | out[ol++] = '\r'; 856 | out[ol++] = '\n'; 857 | continue; 858 | case 8: 859 | if (is_upper) { // rpt 860 | int count = readCount(in, &bit_no, len); 861 | count += 4; 862 | char rpt_c = out[ol - 1]; 863 | while (count--) 864 | out[ol++] = rpt_c; 865 | } else { 866 | out[ol++] = '\n'; 867 | } 868 | continue; 869 | case 10: 870 | continue; 871 | } 872 | } 873 | } 874 | out[ol++] = c; 875 | } 876 | 877 | return ol; 878 | 879 | } 880 | 881 | int is_empty(const char *s) { 882 | while (*s != '\0') { 883 | if (!isspace((unsigned char)*s)) 884 | return 0; 885 | s++; 886 | } 887 | return 1; 888 | } 889 | 890 | // From https://stackoverflow.com/questions/19758270/read-varint-from-linux-sockets#19760246 891 | // Encode an unsigned 64-bit varint. Returns number of encoded bytes. 892 | // 'buffer' must have room for up to 10 bytes. 893 | int encode_unsigned_varint(uint8_t *buffer, uint64_t value) { 894 | int encoded = 0; 895 | do { 896 | uint8_t next_byte = value & 0x7F; 897 | value >>= 7; 898 | if (value) 899 | next_byte |= 0x80; 900 | buffer[encoded++] = next_byte; 901 | } while (value); 902 | return encoded; 903 | } 904 | 905 | uint64_t decode_unsigned_varint(const uint8_t *data, int *decoded_bytes) { 906 | int i = 0; 907 | uint64_t decoded_value = 0; 908 | int shift_amount = 0; 909 | do { 910 | decoded_value |= (uint64_t)(data[i] & 0x7F) << shift_amount; 911 | shift_amount += 7; 912 | } while ((data[i++] & 0x80) != 0); 913 | *decoded_bytes = i; 914 | return decoded_value; 915 | } 916 | 917 | void print_string_as_hex(char *in, int len) { 918 | 919 | int l; 920 | byte bit; 921 | printf("String in hex:\n"); 922 | for (l=0; l>(7-l%8))&0x01; 940 | printf("%d", bit); 941 | if (l%8 == 7) printf(" "); 942 | } 943 | printf("\n"); 944 | 945 | } 946 | 947 | uint32_t getTimeVal() { 948 | #ifdef _MSC_VER 949 | return GetTickCount() * 1000; 950 | #else 951 | struct timeval tv; 952 | gettimeofday(&tv, NULL); 953 | return (tv.tv_sec * 1000000) + tv.tv_usec; 954 | #endif 955 | } 956 | 957 | double timedifference(uint32_t t0, uint32_t t1) { 958 | double ret = t1; 959 | ret -= t0; 960 | ret /= 1000; 961 | return ret; 962 | } 963 | 964 | int main(int argv, char *args[]) { 965 | 966 | char cbuf[1024]; 967 | char dbuf[1024]; 968 | long len, tot_len, clen, ctot, dlen, l; 969 | float perc; 970 | FILE *fp, *wfp; 971 | int bytes_read; 972 | char c_in; 973 | uint32_t tStart; 974 | 975 | tStart = getTimeVal(); 976 | 977 | if (argv == 4 && strcmp(args[1], "-c") == 0) { 978 | tot_len = 0; 979 | ctot = 0; 980 | fp = fopen(args[2], "rb"); 981 | if (fp == NULL) { 982 | perror(args[2]); 983 | return 1; 984 | } 985 | wfp = fopen(args[3], "wb+"); 986 | if (wfp == NULL) { 987 | perror(args[3]); 988 | return 1; 989 | } 990 | do { 991 | bytes_read = fread(cbuf, 1, sizeof(cbuf), fp); 992 | if (bytes_read > 0) { 993 | clen = unishox1_compress(cbuf, bytes_read, dbuf, NULL); 994 | ctot += clen; 995 | tot_len += bytes_read; 996 | if (clen > 0) { 997 | fputc(clen >> 8, wfp); 998 | fputc(clen & 0xFF, wfp); 999 | if (clen != fwrite(dbuf, 1, clen, wfp)) { 1000 | perror("fwrite"); 1001 | return 1; 1002 | } 1003 | } 1004 | } 1005 | } while (bytes_read > 0); 1006 | perc = (tot_len-ctot); 1007 | perc /= tot_len; 1008 | perc *= 100; 1009 | printf("\nBytes (Compressed/Original=Savings%%): %ld/%ld=", ctot, tot_len); 1010 | printf("%.2f%%\n", perc); 1011 | } else 1012 | if (argv == 4 && strcmp(args[1], "-d") == 0) { 1013 | fp = fopen(args[2], "rb"); 1014 | if (fp == NULL) { 1015 | perror(args[2]); 1016 | return 1; 1017 | } 1018 | wfp = fopen(args[3], "wb+"); 1019 | if (wfp == NULL) { 1020 | perror(args[3]); 1021 | return 1; 1022 | } 1023 | do { 1024 | //memset(dbuf, 0, sizeof(dbuf)); 1025 | int len_to_read = fgetc(fp) << 8; 1026 | len_to_read += fgetc(fp); 1027 | bytes_read = fread(dbuf, 1, len_to_read, fp); 1028 | if (bytes_read > 0) { 1029 | dlen = unishox1_decompress(dbuf, bytes_read, cbuf, NULL); 1030 | if (dlen > 0) { 1031 | if (dlen != fwrite(cbuf, 1, dlen, wfp)) { 1032 | perror("fwrite"); 1033 | return 1; 1034 | } 1035 | } 1036 | } 1037 | } while (bytes_read > 0); 1038 | } else 1039 | if (argv == 4 && (strcmp(args[1], "-g") == 0 || 1040 | strcmp(args[1], "-G") == 0)) { 1041 | if (strcmp(args[1], "-g") == 0) 1042 | to_match_repeats = 0; 1043 | fp = fopen(args[2], "r"); 1044 | if (fp == NULL) { 1045 | perror(args[2]); 1046 | return 1; 1047 | } 1048 | sprintf(cbuf, "%s.h", args[3]); 1049 | wfp = fopen(cbuf, "w+"); 1050 | if (wfp == NULL) { 1051 | perror(args[3]); 1052 | return 1; 1053 | } 1054 | tot_len = 0; 1055 | ctot = 0; 1056 | struct us_lnk_lst *cur_line = NULL; 1057 | fputs("#ifndef __", wfp); 1058 | fputs(args[3], wfp); 1059 | fputs("_UNISHOX1_COMPRESSED__\n", wfp); 1060 | fputs("#define __", wfp); 1061 | fputs(args[3], wfp); 1062 | fputs("_UNISHOX1_COMPRESSED__\n", wfp); 1063 | int line_ctr = 0; 1064 | int max_len = 0; 1065 | while (fgets(cbuf, sizeof(cbuf), fp) != NULL) { 1066 | // compress the line and look in previous lines 1067 | // add to linked list 1068 | len = strlen(cbuf); 1069 | if (cbuf[len - 1] == '\n' || cbuf[len - 1] == '\r') { 1070 | len--; 1071 | cbuf[len] = 0; 1072 | } 1073 | if (is_empty(cbuf)) 1074 | continue; 1075 | if (len > 0) { 1076 | struct us_lnk_lst *ll; 1077 | ll = cur_line; 1078 | cur_line = (struct us_lnk_lst *) malloc(sizeof(struct us_lnk_lst)); 1079 | cur_line->data = (char *) malloc(len + 1); 1080 | strncpy(cur_line->data, cbuf, len); 1081 | cur_line->previous = ll; 1082 | clen = unishox1_compress(cbuf, len, dbuf, cur_line); 1083 | if (clen > 0) { 1084 | perc = (len-clen); 1085 | perc /= len; 1086 | perc *= 100; 1087 | //print_compressed(dbuf, clen); 1088 | printf("len: %ld/%ld=", clen, len); 1089 | printf("%.2f %s\n", perc, cbuf); 1090 | tot_len += len; 1091 | ctot += clen; 1092 | char short_buf[strlen(args[3]) + 100]; 1093 | snprintf(short_buf, sizeof(short_buf), "const byte %s_%d[] PROGMEM = {", args[3], line_ctr++); 1094 | fputs(short_buf, wfp); 1095 | int len_len = encode_unsigned_varint((byte *) short_buf, clen); 1096 | for (int i = 0; i < len_len; i++) { 1097 | snprintf(short_buf, 10, "%u, ", (byte) short_buf[i]); 1098 | fputs(short_buf, wfp); 1099 | } 1100 | for (int i = 0; i < clen; i++) { 1101 | if (i) { 1102 | strcpy(short_buf, ", "); 1103 | fputs(short_buf, wfp); 1104 | } 1105 | snprintf(short_buf, 6, "%u", (byte) dbuf[i]); 1106 | fputs(short_buf, wfp); 1107 | } 1108 | strcpy(short_buf, "};\n"); 1109 | fputs(short_buf, wfp); 1110 | } 1111 | if (len > max_len) 1112 | max_len = len; 1113 | dlen = unishox1_decompress(dbuf, clen, cbuf, cur_line); 1114 | cbuf[dlen] = 0; 1115 | printf("\n%s\n", cbuf); 1116 | } 1117 | } 1118 | perc = (tot_len-ctot); 1119 | perc /= tot_len; 1120 | perc *= 100; 1121 | printf("\nBytes (Compressed/Original=Savings%%): %ld/%ld=", ctot, tot_len); 1122 | printf("%.2f%%\n", perc); 1123 | char short_buf[strlen(args[3]) + 100]; 1124 | snprintf(short_buf, sizeof(short_buf), "const byte * const %s[] PROGMEM = {", args[3]); 1125 | fputs(short_buf, wfp); 1126 | for (int i = 0; i < line_ctr; i++) { 1127 | if (i) { 1128 | strcpy(short_buf, ", "); 1129 | fputs(short_buf, wfp); 1130 | } 1131 | snprintf(short_buf, strlen(args[3]) + 15, "%s_%d", args[3], i); 1132 | fputs(short_buf, wfp); 1133 | } 1134 | strcpy(short_buf, "};\n"); 1135 | fputs(short_buf, wfp); 1136 | snprintf(short_buf, sizeof(short_buf), "#define %s_line_count %d\n", args[3], line_ctr); 1137 | fputs(short_buf, wfp); 1138 | snprintf(short_buf, sizeof(short_buf), "#define %s_max_len %d\n", args[3], max_len); 1139 | fputs(short_buf, wfp); 1140 | fputs("#endif\n", wfp); 1141 | } else 1142 | if (argv == 2) { 1143 | len = strlen(args[1]); 1144 | //printf("Len:%ld\n", len); 1145 | //print_string_as_hex(args[1], len); 1146 | memset(cbuf, 0, sizeof(cbuf)); 1147 | ctot = unishox1_compress(args[1], len, cbuf, NULL); 1148 | print_compressed(cbuf, ctot); 1149 | memset(dbuf, 0, sizeof(dbuf)); 1150 | dlen = unishox1_decompress(cbuf, ctot, dbuf, NULL); 1151 | dbuf[dlen] = 0; 1152 | printf("\nDecompressed: %s\n", dbuf); 1153 | //print_compressed(dbuf, dlen); 1154 | perc = (len-ctot); 1155 | perc /= len; 1156 | perc *= 100; 1157 | printf("\nBytes (Compressed/Original=Savings%%): %ld/%ld=", ctot, len); 1158 | printf("%.2f%%\n", perc); 1159 | } else { 1160 | printf("Unishox (byte format version: %s)\n", UNISHOX_VERSION); 1161 | printf("---------------------------------\n"); 1162 | printf("Usage: unishox1 \"string\" or unishox1 [action] [in_file] [out_file] [encoding]\n"); 1163 | printf("\n"); 1164 | printf("Actions:\n"); 1165 | printf(" -c compress\n"); 1166 | printf(" -d decompress\n"); 1167 | printf(" -g generate C header file\n"); 1168 | printf(" -G generate C header file using additional compression (slower)\n"); 1169 | return 1; 1170 | } 1171 | 1172 | printf("\nElapsed: %0.3lf ms\n", timedifference(tStart, getTimeVal())); 1173 | 1174 | return 0; 1175 | 1176 | } 1177 | #pragma GCC diagnostic pop 1178 | -------------------------------------------------------------------------------- /src/unishox1.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * Copyright (C) 2019 Siara Logics (cc) 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | * @author Arundale R. 18 | * 19 | */ 20 | #ifndef unishox1 21 | #define unishox1 22 | 23 | struct us_lnk_lst { 24 | char *data; 25 | struct us_lnk_lst *previous; 26 | }; 27 | 28 | extern int unishox1_compress(const char *in, int len, char *out, struct us_lnk_lst *prev_lines); 29 | extern int unishox1_decompress(const char *in, int len, char *out, struct us_lnk_lst *prev_lines); 30 | 31 | #endif 32 | --------------------------------------------------------------------------------