├── LICENSE.md ├── dub.json ├── source ├── api_user.d ├── example_app.d ├── frail_sql_parser.d ├── layout2dot.d ├── misc.d ├── sqlited.d ├── sqlstuff.d ├── test.d ├── utils.d └── varint.d └── views ├── test-2.3.sqlite ├── test.s3db ├── test2.s3db ├── test3.s3db ├── test4.s3db └── test_chaged.s3db /LICENSE.md: -------------------------------------------------------------------------------- 1 | Boost Software License - Version 1.0 - August 17th, 2003 2 | 3 | Permission is hereby granted, free of charge, to any person or organization 4 | obtaining a copy of the software and accompanying documentation covered by 5 | this license (the "Software") to use, reproduce, display, distribute, 6 | execute, and transmit the Software, and to prepare derivative works of the 7 | Software, and to permit third-parties to whom the Software is furnished to 8 | do so, all subject to the following: 9 | 10 | The copyright notices in the Software and this entire statement, including 11 | the above license grant, this restriction and the following disclaimer, 12 | must be included in all copies of the Software, in whole or in part, and 13 | all derivative works of the Software, unless such copies or derivative 14 | works are solely in the form of machine-executable object code generated by 15 | a source language processor. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 20 | SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE 21 | FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 22 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /dub.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqlite-d", 3 | "description": "A native SQLite Database reader", 4 | "copyright": "Copyright © 2015, Stefan Koch", 5 | "authors": ["Stefan Koch"], 6 | "license" : "boost", 7 | "dependencies": { 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /source/api_user.d: -------------------------------------------------------------------------------- 1 | import sqlited; 2 | 3 | void _main() { 4 | // load a database form file; 5 | Database db = Database("yourdb.sq3"); 6 | 7 | // MyStruct[] deserialized = db.table("MySerialisationTable").deserialize!MyStruct; 8 | // list all tables in the db 9 | // string[] tables = db.tables; 10 | 11 | // auto table = db.table("MyTable"); 12 | 13 | // auto rows = table.findRows!("(x) => x == 5"); 14 | 15 | } 16 | -------------------------------------------------------------------------------- /source/example_app.d: -------------------------------------------------------------------------------- 1 | //import std.stdio; 2 | import sqlited; 3 | import sqlited : isIndex; 4 | import utils; 5 | import misc; 6 | import std.conv; 7 | import std.stdio; 8 | import core.memory; 9 | 10 | import std.algorithm : map, filter, count, reduce, each; 11 | import std.range : join, takeOne; 12 | //import std.typecons : tuple; 13 | static immutable db4 = cast(immutable) Database(cast(ubyte[]) import("test4.s3db"), ""); 14 | 15 | int main_(string[] args) { 16 | string filename = (args.length > 1 ? args[1] : "views/test-2.3.sqlite"); 17 | auto pageNr = (args.length > 2 ? parse!(int)(args[2]) : 0); 18 | writefln("opening file %s", filename); 19 | auto db = new Database(filename); 20 | 21 | Database.MasterTableSchema[] schemas; 22 | 23 | auto test_db = Database("views/test-2.3.sqlite"); 24 | schemas = readRows!(r => r.deserialize!(Database.MasterTableSchema))(db.rootPage, db.pages); 25 | if (pageNr) { 26 | uint x; 27 | foreach(row;Table(db.pages, cast(uint)pageNr)) { 28 | // row.column(0).apply!writeln; 29 | // foreach(col;row.columns) { 30 | // col!(pl => pl.writeln); 31 | // } 32 | x++; 33 | } 34 | x.writeln(); 35 | } else { 36 | foreach(schema;schemas.filter!(s => s.type == "table")) { 37 | writeln(schema.name, ":", schema.rootPage); 38 | } 39 | 40 | } 41 | struct Town { 42 | uint PK_UID; 43 | string name; 44 | uint peoples; 45 | uint localc; 46 | uint country; 47 | uint region; 48 | /* Geomerty point */ 49 | } 50 | Town[] towns; 51 | towns = readRows!(r => r.deserialize!Town)(test_db.table("Towns")); 52 | 53 | foreach(town;towns) { 54 | // writeln(town); 55 | } 56 | 57 | 58 | // if (db !is null) { 59 | // writeln("it appears to be a database"); 60 | import std.datetime; 61 | StopWatch sw; 62 | uint x; 63 | enum times = 2*4096; 64 | foreach(_; 0 .. times) { 65 | string result; 66 | x = 0; 67 | sw.start; 68 | foreach(row;(db4).table("Album")) { 69 | x++; 70 | row.column(1).getAs!string; 71 | } 72 | sw.stop(); 73 | } 74 | 75 | // foreach(_;test_db.table("Regions")) {} 76 | 77 | writeln("Getting all ", x, " entries of column 1 in table Album ", times, " times took ", sw.peek().msecs, "msecs"); 78 | // writeln(db4.table("Artist")); 79 | return 0; 80 | // } else { 81 | // writeln("invalid database or header corrupted"); 82 | // } 83 | assert(0); 84 | 85 | // readln(); 86 | } 87 | -------------------------------------------------------------------------------- /source/frail_sql_parser.d: -------------------------------------------------------------------------------- 1 | // "CREATE TABLE `table_name` ( 2 | 3 | /*CREATE TABLE spatial_ref_sys ( 4 | srid INTEGER NOT NULL PRIMARY KEY, 5 | auth_name VARCHAR(256) NOT NULL, 6 | auth_srid INTEGER NOT NULL, 7 | ref_sys_name VARCHAR(256), 8 | proj4text VARCHAR(2048) NOT NULL), 9 | */ 10 | import std.algorithm; 11 | alias lengthType = uint; 12 | pure : 13 | uint countUntil(SR, T...)(SR s, T matches) 14 | { 15 | uint count = 0; 16 | foreach (c;s) 17 | { 18 | foreach(m;matches) 19 | { 20 | if (m == c) 21 | return count; 22 | } 23 | ++count; 24 | } 25 | return count; 26 | } 27 | 28 | auto getDelim(char c) { 29 | switch(c) { 30 | case '"' : 31 | return '"'; 32 | case '[' : 33 | return ']'; 34 | case '`' : 35 | return '`'; 36 | default : 37 | // if we don't have a recognized delimiter... delimit by Space 38 | // THIS IS A HACK!!! 39 | return ' '; 40 | } 41 | } 42 | import sqlstuff; 43 | void skipWhiteSpace(string _string, uint* pos) { 44 | 45 | import std.ascii : isWhite; 46 | uint _pos = *pos; 47 | while(isWhite(_string[_pos++]) && _pos < _string.length) {} 48 | *pos = _pos - 1; 49 | } 50 | 51 | 52 | auto parseCreateTable(const string sql) pure { 53 | ColumInfo[] columInfos; 54 | uint pos = cast(uint) "CREATE TABLE".length; 55 | sql.skipWhiteSpace(&pos); 56 | 57 | auto delim = getDelim(sql[pos]); 58 | pos += (delim == ' ' ? 0 : 1); 59 | auto strlen = cast(uint)sql[pos .. $].countUntil(delim); 60 | 61 | string tableName = sql[pos .. pos + strlen]; 62 | pos += strlen + (delim == ' ' ? 0 : 1); 63 | sql.skipWhiteSpace(&pos); 64 | debug { 65 | import std.stdio; 66 | // writeln("tableName :",tableName); 67 | // writeln(sql[pos .. $]); 68 | } 69 | assert(sql[pos] == '('); 70 | pos++; 71 | while(sql[pos] != ')') { 72 | auto res = parseColum(sql[pos .. $]); 73 | columInfos ~= res.colum; 74 | pos += res.length; 75 | pos += (sql[pos] == ',' ? 1 : 0); 76 | } 77 | 78 | return TableInfo(tableName, columInfos); 79 | } 80 | 81 | 82 | 83 | auto parseColum(const string sql) pure { 84 | enum KeywordEnum { 85 | _, 86 | notNull, 87 | primaryKey, 88 | autoincrement, 89 | // unique, 90 | } 91 | struct Result { 92 | ColumInfo colum; 93 | uint length; 94 | } 95 | Result result; 96 | 97 | sql.skipWhiteSpace(&result.length); 98 | auto delim = getDelim(sql[result.length]); 99 | result.length += (delim == ' ' ? 0 : 1); 100 | int strlen = cast(int) sql[result.length .. $].countUntil(delim); 101 | 102 | result.colum.name = sql[result.length .. result.length + strlen]; 103 | result.length += strlen + (delim == ' ' ? 0 : 1); 104 | sql.skipWhiteSpace(&result.length); 105 | 106 | delim = getDelim(sql[result.length]); 107 | strlen = cast(int) sql[result.length .. $].countUntil(delim,',','\n'); 108 | 109 | //TODO make the ')' work as final delimiter; 110 | // because this is still a nasty hack! 111 | // a nasty nasty HACK!!! 112 | //so if I would eat the the final ')' truncate the strlen before the final ')' 113 | import std.range : retro; 114 | auto lastParen = cast(int)sql.retro.countUntil(')'); 115 | if (sql.length - result.length - strlen <= lastParen) { 116 | strlen -= 1; 117 | } 118 | // the code above is a nasty nasty nasty HACK!!! 119 | 120 | result.colum.typeName = sql[result.length .. result.length + strlen]; 121 | result.length += strlen + (delim == ' ' ? 0 : 1); 122 | sql.skipWhiteSpace(&result.length); 123 | 124 | 125 | debug { 126 | import std.stdio; 127 | if (!__ctfe) { 128 | writeln("Before while:" ,result.colum , sql[result.length .. $]); 129 | } 130 | } 131 | while (sql[result.length] != ',' && sql[result.length] != ')') { 132 | import std.ascii; 133 | if (auto kw = cast(KeywordEnum)sql[result.length .. $].map!(c => toUpper(c)).startsWith("NOT NULL","PRIMARY KEY","AUTOINCREMENT")) { 134 | final switch (kw) with(KeywordEnum) { 135 | case _ : assert(0); // cannot ever happen 136 | case notNull : 137 | assert(result.colum.notNull == false); 138 | result.colum.notNull = true; 139 | result.length += "NOT NULL".length; 140 | break; 141 | case primaryKey : 142 | assert(result.colum.primaryKey == false); 143 | result.colum.primaryKey = true; 144 | result.length += "PRIMARY KEY".length; 145 | break; 146 | case autoincrement : 147 | assert(result.colum.autoincrement == false); 148 | result.colum.autoincrement = true; 149 | result.length += "AUTOINCREMENT".length; 150 | break; 151 | } 152 | sql.skipWhiteSpace(&result.length); 153 | debug { 154 | import std.stdio; 155 | if (!__ctfe) { 156 | // writeln(result); 157 | } 158 | } 159 | continue ; 160 | } else { 161 | debug { 162 | import std.stdio; 163 | if (!__ctfe) { 164 | writeln(result); 165 | writeln(sql[result.length .. $]); 166 | } 167 | } 168 | assert(0, "Unhandeled Keyword or something"); 169 | } 170 | 171 | 172 | } 173 | 174 | 175 | return result; 176 | 177 | } 178 | -------------------------------------------------------------------------------- /source/layout2dot.d: -------------------------------------------------------------------------------- 1 | module layout2dot; 2 | import sqlited; 3 | import utils; 4 | import std.conv : to; 5 | import misc; 6 | 7 | string TreeLayoutToDot(Database db) { 8 | string result = "digraph table {"; 9 | 10 | Database.MasterTableSchema[] schemas; 11 | schemas = readRows!(r => r.deserialize!(Database.MasterTableSchema))(db.rootPage, 12 | db.pages); 13 | result ~= TreeLayoutToDot(db.rootPage, db.pages, 1); 14 | foreach (schema; schemas) { 15 | if (schema.rootPage) { 16 | result ~= "\n" ~ "\"Root\"" ~ " -> " ~ '"' ~ toQuotedString(schema.rootPage) ~ '"' ~ "\n"; 17 | result ~= TreeLayoutToDot(db.pages[schema.rootPage - 1], db.pages, schema.rootPage) ~ "\n"; 18 | } 19 | } 20 | 21 | return result ~ "}\n"; 22 | } 23 | 24 | string toQuotedString(uint i) { 25 | return "\"" ~ to!string(i) ~ "\""; 26 | } 27 | 28 | string TreeLayoutToDot(Database.BTreePage page, Database.PageRange pages, uint rootPage) { 29 | string pref = "\n" ~ toQuotedString(rootPage) ~ " -> "; 30 | string result; 31 | auto cpa = page.getCellPointerArray(); 32 | 33 | switch (page.pageType) { 34 | case Database.BTreePageType.tableInteriorPage: { 35 | foreach (cp; cpa) { 36 | uint nextPage = BigEndian!uint(page.page[cp .. cp + uint.sizeof]); 37 | result ~= pref ~ toQuotedString(nextPage); 38 | result ~= TreeLayoutToDot(pages[nextPage - 1], pages, nextPage); 39 | } 40 | 41 | uint nextPage = page.header._rightmostPointer; 42 | result ~= pref ~ toQuotedString(nextPage); 43 | result ~= TreeLayoutToDot(pages[nextPage - 1], pages, nextPage); 44 | } 45 | break; 46 | case Database.BTreePageType.tableLeafPage: { 47 | 48 | } 49 | break; 50 | default: { 51 | } 52 | 53 | } 54 | return result; 55 | } 56 | -------------------------------------------------------------------------------- /source/misc.d: -------------------------------------------------------------------------------- 1 | module misc; 2 | // Copyright Stefan Koch 2015 - 2018. 3 | // Distributed under the Boost Software License, Version 1.0. 4 | // (See accompanying file LICENSE.md or copy at 5 | // http://www.boost.org/LICENSE_1_0.txt) 6 | 7 | import sqlited; 8 | import utils; 9 | import std.traits; 10 | 11 | struct_type deserialize(struct_type)(Database.Row r) if (is(struct_type == struct)) { 12 | struct_type instance; 13 | uint ctr; 14 | foreach (member; __traits(derivedMembers, struct_type)) { 15 | alias type = typeof(__traits(getMember, instance, member)); 16 | static if (!is(type == function)) { 17 | __traits(getMember, instance, member) = r.getAs!(type)(ctr++); 18 | } 19 | } 20 | return instance; 21 | } 22 | 23 | import std.typecons; 24 | 25 | struct Table { 26 | const Database.PageRange pages; 27 | const uint rootPage; 28 | 29 | int opApply(scope int delegate(const Database.Row r) dg) { 30 | readRowDg!dg(this); 31 | return 0; 32 | } 33 | 34 | struct_type[] deserialize(struct_type)() if (is(struct_type == struct)) { 35 | struct_type[] result; 36 | foreach(row;this) { 37 | result ~= row.deserialize!struct_type; 38 | } 39 | return result; 40 | } 41 | 42 | 43 | } 44 | 45 | Table table(const Database db, in string tableName) pure { 46 | uint rootPage; 47 | 48 | readRows!( 49 | (r) { 50 | if (r.column(0).getAs!string == "table" && 51 | r.column(1).getAs!string == tableName) { 52 | rootPage = r.column(3).getAs!uint - 1; 53 | } 54 | //assert(rootPage, "table not found"); 55 | })(db.pages[0], db.pages); 56 | 57 | 58 | return Table(db.pages, rootPage); 59 | } 60 | 61 | 62 | /// usage : table.select("name","surname").where!("age","sex", (age, sex) => sex.as!Sex == Sex.female, age.as!uint < 40)) 63 | /// or table.select("name").where!((type) => type.as!string == "table")("type").as!string; 64 | /// or join().select() 65 | /+ 66 | auto where(S,T...)(S selectResult, T ) 67 | 68 | auto where(S,T...)(S selectResult) { 69 | foreach(i,Element;T) { 70 | static if (i == 0) { 71 | static assert(is(Element == delegeate) || is(Element == function), 72 | "first template argument to where has to be a delegate or a function"); 73 | static assert() 74 | } 75 | } 76 | } 77 | 78 | auto SQL(SQLElements...)(Database db) { 79 | // static assert (allStatiesfy(isSQLElement!SQLElements)) 80 | foreach(elem;SQLElements) { 81 | static if (isSelect!elem) { 82 | //assert that there is just one select 83 | } else static if (isWhere!elem) { 84 | 85 | } 86 | } 87 | } 88 | +/ 89 | /// handlePage is used to itterate over interiorPages transparently 90 | /+ 91 | void* handlePageF(Database.BTreePage page, 92 | Database.PageRange pages, 93 | void* function(Database.BTreePage, Database.PageRange, void*) pageHandlerF, 94 | void* initialState = null) { 95 | handleRow!( 96 | (page, pages) => initialState = pageHandlerF(page, pages, initialState) 97 | )(page, pages); 98 | 99 | return initialState; 100 | } 101 | +/ 102 | template pageHandlerTypeP(alias pageHandler) { 103 | alias pageHandlerTypeP = typeof((cast(const)Database.BTreePage.init)); 104 | } 105 | 106 | template pageHandlerTypePP(alias pageHandler) { 107 | alias pageHandlerTypePP = typeof(pageHandler(cast(const)Database.BTreePage.init, cast(const)Database.PageRange.init)); 108 | } 109 | 110 | template rowHandlerTypeR(alias rowHandler) { 111 | alias rowHandlerTypeR = typeof(rowHandler(cast(const)Database.BTreePage.Row.init)); 112 | } 113 | 114 | template rowHandlerTypeRP(alias rowHandler) { 115 | alias rowHandlerTypeRP = typeof(rowHandler(cast(const)Database.BTreePage.Row.init, cast(const)Database.PageRange.init)); 116 | } 117 | 118 | template rowHandlerReturnType(alias rowHandler) { 119 | alias typeR = rowHandlerTypeR!rowHandler; 120 | 121 | static if (is(typeR)) { 122 | alias rowHandlerReturnType = typeR; 123 | } else { 124 | static assert(0, "pageHandler has to be callable with (Row)" ~ typeof(rowHandler).stringof); 125 | } 126 | } 127 | 128 | template pageHandlerReturnType(alias pageHandler) { 129 | alias typePP = pageHandlerTypePP!pageHandler; 130 | alias typeP = pageHandlerTypeP!pageHandler; 131 | 132 | static if (is(typePP)) { 133 | alias pageHandlerReturnType = typePP; 134 | } else static if (is(typeP)) { 135 | alias pageHandlerReturnType = typeP; 136 | } else { 137 | static assert(0, "pageHandler has to be callable with (BTreePage) or (BTreePage, PageRange)" ~ typeof(pageHandler).stringof); 138 | } 139 | } 140 | static assert (is(pageHandlerReturnType!((page, pages) => page))); 141 | 142 | auto readRows(alias RowHandler)(const Table table) { 143 | return readRows!(RowHandler)(table.pages[cast(uint)table.rootPage], table.pages); 144 | } 145 | 146 | int readRowDg(alias dg)(const Table table) { 147 | readRows!((r) {dg(r);})(table); 148 | return 0; 149 | } 150 | 151 | RR readRows(alias rowHandler, RR = rowHandlerReturnType!(rowHandler)[])(const Database.BTreePage page, 152 | const Database.PageRange pages) { 153 | RR returnArray; 154 | enum noReturn = is(RR == void[]); 155 | enum isPure = is(typeof((){void _() pure {rowHandler(cast(const)Database.Row.init);}}())); 156 | 157 | // pragma(msg, isPure); 158 | // static assert(isPure); 159 | readRows!rowHandler(page, pages, returnArray); 160 | return returnArray; 161 | } 162 | 163 | 164 | /// handlePage is used to itterate over interiorPages transparently 165 | void readRows(alias rowHandler,bool writable = false, RR)(const Database.BTreePage page, 166 | const Database.PageRange pages, ref RR returnRange) { 167 | alias hrt = rowHandlerReturnType!(rowHandler); 168 | alias defaultReturnRangeType = hrt[]; 169 | enum isPure = is(typeof((){void _() pure {rowHandler(cast(const)Database.Row.init);}}())); 170 | enum noReturn = is(hrt == void) || is(hrt == typeof(null)); 171 | 172 | static assert(is(hrt), "rowHandler has to be callable with (Row)" ~ typeof(rowHandler).stringof); 173 | 174 | auto cpa = page.getCellPointerArray(); 175 | 176 | switch (page.pageType) with (Database.BTreePage.BTreePageType) { 177 | 178 | case tableLeafPage: { 179 | foreach(cp;cpa) { 180 | static if (noReturn) { 181 | rowHandler(page.getRow(cp, pages, page.pageType)); 182 | } else { 183 | static if (is (RR == defaultReturnRangeType)) { 184 | returnRange ~= rowHandler(page.getRow(cp, pages, page.pageType)); 185 | } else { 186 | returnRange.put(rowHandler(page.getRow(cp, pages, page.pageType))); 187 | } 188 | } 189 | } 190 | } break; 191 | 192 | 193 | case tableInteriorPage: { 194 | foreach(cp;cpa) { 195 | readRows!rowHandler(pages[BigEndian!uint(page.page[cp .. cp + uint.sizeof]) - 1], pages, returnRange); 196 | //TODO Read The Key! 197 | } 198 | 199 | readRows!rowHandler(pages[page.header._rightmostPointer - 1], pages, returnRange); 200 | 201 | } break; 202 | 203 | case indexInteriorPage: { 204 | 205 | foreach(cp_;cpa) { 206 | ushort cp = cast(ushort)(cp_ + uint.sizeof); 207 | 208 | static if (noReturn) { 209 | rowHandler(page.getRow(cp, pages, page.pageType)); 210 | } else { 211 | static if (is (RR == defaultReturnRangeType)) { 212 | returnRange ~= rowHandler(page.getRow(cp, pages, page.pageType)); 213 | } else { 214 | returnRange.put(rowHandler(page.getRow(cp, pages, page.pageType))); 215 | } 216 | } 217 | 218 | readRows!rowHandler(pages[BigEndian!uint(page.page[cp_ .. cp]) - 1], pages, returnRange); 219 | } 220 | 221 | readRows!rowHandler(pages[page.header._rightmostPointer - 1], pages, returnRange); 222 | 223 | 224 | } break ; 225 | 226 | case indexLeafPage: { 227 | 228 | foreach(cp;cpa) { 229 | static if (noReturn) { 230 | rowHandler(page.getRow(cp, pages, page.pageType)); 231 | } else { 232 | static if (is (RR == defaultReturnRangeType)) { 233 | returnRange ~= rowHandler(page.getRow(cp, pages, page.pageType)); 234 | } else { 235 | returnRange.put(rowHandler(page.getRow(cp, pages, page.pageType))); 236 | } 237 | } 238 | } 239 | } break ; 240 | 241 | default: 242 | assert(0, "empty pages are not handled by readRows!"); 243 | } 244 | 245 | return ; 246 | 247 | } 248 | -------------------------------------------------------------------------------- /source/sqlited.d: -------------------------------------------------------------------------------- 1 | module sqlited; 2 | 3 | // Copyright Stefan Koch 2015 - 2018. 4 | // Distributed under the Boost Software License, Version 1.0. 5 | // (See accompanying file LICENSE.md or copy at 6 | // http://www.boost.org/LICENSE_1_0.txt) 7 | 8 | /**************************** 9 | * SQLite-D SQLite 3 Reader * 10 | * By Stefan Koch 2016 * 11 | ****************************/ 12 | 13 | import utils; 14 | import varint; 15 | 16 | struct Database { 17 | string dbFilename; 18 | const ubyte[] data; 19 | alias Row = BTreePage.Row; 20 | alias BTreePageType = BTreePage.BTreePageType; 21 | 22 | SQLiteHeader _cachedHeader; 23 | 24 | SQLiteHeader header() pure const { 25 | return _cachedHeader; 26 | } 27 | 28 | BTreePage rootPage() pure const { 29 | return pages[0]; 30 | } 31 | 32 | PageRange pages() pure const { 33 | return PageRange(cast(const)data, header.pageSize, 34 | usablePageSize, cast(const uint)(data.length / header.pageSize)); 35 | } 36 | 37 | const(uint) usablePageSize() pure const { 38 | return header.pageSize - header.reserved; 39 | } 40 | 41 | struct PageRange { 42 | const ubyte[] data; 43 | const uint pageSize; 44 | const uint usablePageSize; 45 | 46 | const uint numberOfPages; 47 | pure : 48 | @property uint length() const { 49 | return numberOfPages; 50 | } 51 | 52 | BTreePage opIndex(const uint pageNumber) pure const { 53 | assert(pageNumber <= numberOfPages, 54 | "Attempting to go to invalid page"); 55 | 56 | const size_t pageBegin = 57 | (pageSize * pageNumber); 58 | const size_t pageEnd = 59 | (pageSize * pageNumber) + usablePageSize; 60 | 61 | return BTreePage(data[pageBegin .. pageEnd], pageNumber ? 0 : 100); 62 | } 63 | 64 | this(const ubyte[] data, const uint pageSize, 65 | const uint usablePageSize, const uint numberOfPages) pure { 66 | this.data = data; 67 | this.pageSize = pageSize; 68 | this.usablePageSize = usablePageSize; 69 | this.numberOfPages = numberOfPages; 70 | assert(usablePageSize >= 512); 71 | } 72 | 73 | } 74 | 75 | static struct Payload { 76 | alias SerialTypeCodeEnum = SerialTypeCode.SerialTypeCodeEnum; 77 | static struct SerialTypeCode { 78 | alias SerialTypeCodeEnum this; 79 | 80 | static enum SerialTypeCodeEnum : long { 81 | NULL = (0), 82 | int8 = (1), 83 | int16 = (2), 84 | int24 = (3), 85 | int32 = (4), 86 | int48 = (5), 87 | int64 = (6), 88 | float64 = (7), 89 | 90 | bool_false = (8), 91 | bool_true = (9), 92 | 93 | blob = (12), 94 | _string = (13) 95 | } 96 | 97 | this(VarInt v) pure nothrow { 98 | long _v = v; // this could be an int 99 | // also the varint handling would not need to swap 8 bytes 100 | 101 | if (_v > 11) { 102 | if (_v & 1) { 103 | type = SerialTypeCode.SerialTypeCodeEnum._string; 104 | length = (_v - 13) / 2; 105 | } else { 106 | type = SerialTypeCode.SerialTypeCodeEnum.blob; 107 | length = (_v - 12) / 2; 108 | } 109 | } else { 110 | static immutable uint[12] lengthTbl = [0, 1, 2, 3, 111 | 4, 6, 8, 8, 112 | 0, 0, -1, -1 113 | ]; 114 | type = cast(SerialTypeCodeEnum) _v; 115 | length = lengthTbl[_v]; 116 | } 117 | 118 | } 119 | 120 | SerialTypeCodeEnum type; 121 | long length; 122 | } 123 | 124 | union { 125 | byte int8; 126 | short int16; 127 | int int24; 128 | int int32; 129 | long int48; 130 | long int64; 131 | double float64; 132 | const(char)[] _string; 133 | ubyte[] blob; 134 | } 135 | 136 | SerialTypeCode typeCode; 137 | alias length = typeCode.length; 138 | alias type = typeCode.type; 139 | } 140 | 141 | static struct Table { 142 | TableSchema schema; 143 | // Row[] rows; 144 | } 145 | 146 | static align(1) struct SQLiteHeader { 147 | 148 | bool isValid() pure { 149 | return magicString == "SQLite format 3\0"; 150 | } 151 | 152 | align(1): 153 | // ALL NUMBERS ARE BIGENDIAN 154 | static { 155 | enum TextEncoding : uint { 156 | utf8 = bigEndian(1), 157 | utf16le = bigEndian(2), 158 | utf16be = bigEndian(3) 159 | } 160 | 161 | enum SchemaFormat : uint { 162 | _1 = bigEndian(1), 163 | _2 = bigEndian(2), 164 | _3 = bigEndian(3), 165 | _4 = bigEndian(4), 166 | } 167 | 168 | enum FileFormat : ubyte { 169 | legacy = 1, 170 | wal = 2, 171 | } 172 | 173 | struct Freelist { 174 | BigEndian!uint nextPage; 175 | BigEndian!uint leafPointers; // Number if leafPoinrers; 176 | } 177 | 178 | } 179 | 180 | char[16] magicString; 181 | 182 | BigEndian!ushort pageSize; /// between 512 and 32768 or 1 183 | 184 | FileFormat FileFormatWriteVer; 185 | FileFormat FileFormatReadVer; 186 | 187 | ubyte reserved; /// unused bytes at the end of each page 188 | 189 | immutable ubyte maxEmbeddedPayloadFract = 64; 190 | immutable ubyte minEmbeddedPayloadFract = 32; 191 | immutable ubyte leafPayloadFract = 32; 192 | BigEndian!uint fileChangeCounter; 193 | BigEndian!uint sizeInPages; /// fileChangeCounter has to match validForVersion or sizeInPages is invalid 194 | 195 | BigEndian!uint firstFreelistPage; /// Page number of the first freelist trunk page. 196 | BigEndian!uint freelistPages; /// Total number of freelist Pages; 197 | 198 | BigEndian!uint schemaCookie; 199 | SchemaFormat schemaFormatVer; // 1,2,3 or 4 200 | 201 | BigEndian!uint defaultCacheSize; 202 | BigEndian!uint largestRootPage; /// used in incermental and auto vacuum modes. 0 otherwise. 203 | 204 | TextEncoding textEncoding; 205 | 206 | BigEndian!uint userVersion; 207 | 208 | BigEndian!uint incrementalVacuum; /// Non-Zero if on, Zero otherwise; 209 | 210 | BigEndian!uint applicationId; 211 | 212 | BigEndian!uint[5] _reserved; /// Reserved space for future format expansion 213 | 214 | BigEndian!uint validForVersion; 215 | BigEndian!uint sqliteVersion; 216 | 217 | static SQLiteHeader fromArray (const ubyte[] raw) pure { 218 | assert(raw.length >= this.sizeof); 219 | if (__ctfe) { 220 | SQLiteHeader result; 221 | 222 | result.magicString = cast(char[])raw[0 .. 16]; 223 | result.pageSize = raw[16 .. 18]; 224 | result.FileFormatWriteVer = cast(FileFormat)raw[18 .. 19][0]; 225 | result.FileFormatReadVer = cast(FileFormat)raw[19 .. 20][0]; 226 | result.reserved = raw[20 .. 21][0]; 227 | // maxEmbeddedPayloadFract ff. 228 | // are immutable values and do not need to be read 229 | 230 | // result.maxEmbeddedPayloadFract = raw[21 .. 22]; 231 | // result.minEmbeddedPayloadFract = raw[22 .. 23]; 232 | // result.leafPayloadFract = raw[23 .. 24] 233 | 234 | result.fileChangeCounter = raw[24 .. 28]; 235 | result.sizeInPages = raw[28 .. 32]; 236 | result.firstFreelistPage = raw[32 .. 36]; 237 | 238 | return result; 239 | } else { 240 | return *(cast(SQLiteHeader*) raw); 241 | } 242 | } 243 | } 244 | 245 | this(string filename, bool readEntirely = true) { 246 | import std.file; 247 | 248 | auto data = cast(ubyte[]) read(filename); 249 | this(data, filename); 250 | } 251 | /// If you pass null as buffer a new one will be gc-allocated; 252 | this(const ubyte[] buffer, string filename = ":Memory:") pure { 253 | if (buffer is null) { 254 | ubyte[] myBuffer = new ubyte[](1024); 255 | data = myBuffer; 256 | ///TODO write a suitable default header here. 257 | } else { 258 | data = cast(ubyte[])buffer; 259 | } 260 | dbFilename = filename; 261 | _cachedHeader = SQLiteHeader.fromArray(buffer); 262 | assert(_cachedHeader.magicString[0..6] == "SQLite"); 263 | } 264 | 265 | /** 266 | * CREATE TABLE sqlite_master( 267 | * type text, 268 | * name text, 269 | * tbl_name text, 270 | * rootpage integer, 271 | * sql text 272 | * ); 273 | * 274 | */ 275 | static struct MasterTableSchema { 276 | string type; 277 | string name; 278 | string tbl_name; 279 | uint rootPage; 280 | string sql; 281 | } 282 | 283 | static struct TableSchema { 284 | static struct SchemaEntry { 285 | // uint colNumber; 286 | string columnName; 287 | string TypeName; 288 | string defaultValue; 289 | bool isPrimayKey; 290 | bool notNull; 291 | } 292 | } 293 | 294 | 295 | 296 | static struct BTreePage { 297 | 298 | enum BTreePageType : ubyte { 299 | emptyPage = 0, 300 | indexInteriorPage = 2, 301 | tableInteriorPage = 5, 302 | indexLeafPage = 10, 303 | tableLeafPage = 13 304 | } 305 | 306 | 307 | const ubyte[] page; 308 | const uint headerOffset; 309 | 310 | static struct Row { 311 | const PageRange pages; 312 | const uint payloadSize; 313 | const uint rowId; 314 | const uint payloadHeaderSize; 315 | const ubyte[] payloadHeaderBytes; 316 | const ubyte[] payloadStart; 317 | const BTreePageType pageType; 318 | 319 | auto column(const uint colNum) pure const { 320 | auto payloadHeader = PayloadHeader(payloadHeaderBytes); 321 | uint offset; 322 | 323 | foreach(_; 0 .. colNum) { 324 | offset += payloadHeader.front().length; 325 | payloadHeader.popFront(); 326 | } 327 | 328 | auto typeCode = payloadHeader.front(); 329 | uint payloadEnd = cast(uint) (offset + typeCode.length); 330 | 331 | if (payloadStart.length > payloadEnd) { 332 | return extractPayload(payloadStart[offset .. payloadEnd], typeCode); 333 | } else { 334 | auto overflowInfo = OverflowInfo(payloadStart, offset, payloadSize, pages, payloadHeaderSize, pageType); 335 | return extractPayload(&overflowInfo, typeCode, pages); 336 | } 337 | } 338 | } 339 | 340 | 341 | 342 | Row getRow(const ushort cellPointer, const PageRange pages, const BTreePageType pageType) pure const { 343 | uint offset = cellPointer; 344 | import std.algorithm : min; 345 | 346 | auto payloadSize = VarInt(page[offset .. $]); 347 | offset += payloadSize.length; 348 | 349 | ulong rowId; 350 | if (pageType == BTreePageType.tableLeafPage) { 351 | VarInt rowId_v = VarInt(page[offset .. $]); 352 | rowId = rowId_v; 353 | offset += rowId_v.length; 354 | } 355 | 356 | auto payloadHeaderSize = VarInt(page[offset .. page.length]); 357 | uint _payloadHeaderSize = cast(uint)payloadHeaderSize; 358 | 359 | if (_payloadHeaderSize > page.length - offset) { 360 | assert(0, "Overflowing payloadHeaders are currently not handeled"); 361 | } 362 | 363 | auto ph = page[offset + payloadHeaderSize.length .. offset + _payloadHeaderSize]; 364 | offset += _payloadHeaderSize; 365 | // TODO The payloadHeader does not have to be sliced off here 366 | // We can potentially do better of we pass just one buffer to struct Row and slice there. 367 | 368 | return Row(pages, cast(uint) payloadSize, cast(uint) rowId, cast(uint) _payloadHeaderSize , ph, page[offset .. $], pageType); 369 | } 370 | 371 | // string toString(PageRange pages) { 372 | // import std.conv; 373 | // 374 | // auto pageType = header.pageType; 375 | // string result = to!string(pageType); 376 | // 377 | // auto cellPointers = getCellPointerArray(); 378 | // foreach (cp; cellPointers) { 379 | // ubyte* printPtr = cast(ubyte*) base + cp; 380 | // 381 | // final switch (header.pageType) with ( 382 | // BTreePageHeader.BTreePageType) { 383 | // case emptyPage: 384 | // result ~= "This page is Empty or the pointer is bogus\n"; 385 | // break; 386 | // 387 | // case tableLeafPage: { 388 | // auto singlePagePayloadSize = usablePageSize - 35; 389 | // result ~= "singlePagePayloadSize : " ~ ( 390 | // singlePagePayloadSize).to!string ~ "\n"; 391 | // auto payloadSize = VarInt(printPtr); 392 | // result ~= "payloadSize: " ~ (payloadSize).to!string 393 | // ~ "\n"; 394 | // printPtr += payloadSize.length; 395 | // auto rowid = VarInt(printPtr); 396 | // result ~= "rowid: " ~ (rowid).to!string ~ "\n"; 397 | // printPtr += rowid.length; 398 | // 399 | // auto payloadHeaderSize = VarInt(printPtr); 400 | // result ~= "payloadHeaderSize: " ~ (payloadHeaderSize) 401 | // .to!string ~ "\n"; 402 | // 403 | // printPtr += payloadHeaderSize.length; 404 | // 405 | // auto typeCodes = processPayloadHeader(printPtr, 406 | // payloadHeaderSize); 407 | // printPtr += payloadHeaderSize - payloadHeaderSize.length; 408 | // 409 | // import std.algorithm; 410 | // assert(typeCodes.map!(tc => tc.length).sum == payloadSize - payloadHeaderSize); 411 | // 412 | // result ~= "{ "; 413 | // if (payloadSize < singlePagePayloadSize) { 414 | // foreach (typeCode; typeCodes) { 415 | // auto p = extractPayload(printPtr, typeCode); 416 | // result ~= p.apply!( 417 | // v => "\t\"" ~ to!string(v) ~ "\",\n"); 418 | // printPtr += typeCode.length; 419 | // } 420 | // } else { 421 | // auto overflowInfo = OverflowInfo(); 422 | // 423 | // overflowInfo.remainingTotalPayload = cast(uint)payloadSize; 424 | // overflowInfo.payloadOnFirstPage = 425 | // payloadOnPage(cast(uint)(payloadSize)) - cast(uint)payloadHeaderSize; 426 | // 427 | // foreach (typeCode; typeCodes) { 428 | // auto p = extractPayload(&printPtr, typeCode, 429 | // &overflowInfo, pages); 430 | // auto str = p.apply!(v => "\t\"" ~ to!string(v) ~ "\",\n"); 431 | // result ~= str; 432 | // 433 | // } 434 | // } 435 | // 436 | // result ~= " }\n"; 437 | // 438 | // printPtr += payloadSize; 439 | // } 440 | // break; 441 | // 442 | // case tableInteriorPage: { 443 | // BigEndian!uint leftChildPointer = *( 444 | // cast(uint*) printPtr); 445 | // result ~= "nextPagePointer: " ~ (leftChildPointer) 446 | // .to!string ~ "\n"; 447 | // //auto lc = BTreePage(base, usablePageSize, leftChildPointer); 448 | // //result ~= to!string(lc); 449 | // printPtr += uint.sizeof; 450 | // auto integerKey = VarInt(printPtr); 451 | // result ~= "integerKey: " ~ (integerKey).to!string 452 | // ~ "\n"; 453 | // printPtr += integerKey.length; 454 | // } 455 | // 456 | // break; 457 | // 458 | // case indexLeafPage: { 459 | // auto payloadSize = VarInt(cast(ubyte*) printPtr); 460 | // result ~= "payloadSize: " ~ (payloadSize).to!string 461 | // ~ "\n"; 462 | // printPtr += payloadSize.length; 463 | // auto payloadHeaderSize = VarInt(cast(ubyte*) printPtr); 464 | // printPtr += payloadHeaderSize.length; 465 | // result ~= "payloadHeaderSize: " ~ (payloadHeaderSize) 466 | // .to!string ~ "\n"; 467 | // auto typeCodes = processPayloadHeader(printPtr, 468 | // payloadHeaderSize); 469 | // foreach (typeCode; typeCodes) { 470 | // result ~= to!string(typeCode) ~ ", "; 471 | // } 472 | // result ~= "\n"; 473 | // result ~= "rs-phs : " ~ to!string( 474 | // payloadSize - payloadHeaderSize) ~ "\n"; 475 | // auto payload = CArray!char.toArray( 476 | // cast(ubyte*)(printPtr + payloadHeaderSize), 477 | // payloadSize - payloadHeaderSize); 478 | // printPtr += payloadSize; 479 | // result ~= (payload.length) ? (payload).to!string : ""; 480 | // 481 | // } 482 | // break; 483 | // 484 | // case indexInteriorPage: { 485 | // BigEndian!uint leftChildPointer = 486 | // *(cast(uint*) printPtr); 487 | // result ~= "leftChildPinter: " ~ (leftChildPointer) 488 | // .to!string ~ "\n"; 489 | // VarInt payloadSize; 490 | // CArray!ubyte _payload; 491 | // BigEndian!uint _firstOverflowPage; 492 | // //assert(0,"No support for indexInteriorPage"); 493 | // } 494 | // break; 495 | // 496 | // } 497 | // 498 | // } 499 | // return result; 500 | // } 501 | 502 | static align(1) struct BTreePageHeader { 503 | align(1): 504 | // pure : 505 | BTreePageType _pageType; 506 | BigEndian!ushort firstFreeBlock; 507 | BigEndian!ushort cellsInPage; 508 | BigEndian!ushort startCellContantArea; /// 0 is interpreted as 65536 509 | ubyte fragmentedFreeBytes; 510 | BigEndian!uint _rightmostPointer; 511 | 512 | static BTreePageHeader fromArray(const ubyte[] _array) pure { 513 | assert(_array.length >= this.sizeof); 514 | if (__ctfe) { 515 | BTreePageHeader result; 516 | 517 | result._pageType = cast(BTreePageType)_array[0]; 518 | result.firstFreeBlock = _array[1 .. 3]; 519 | result.cellsInPage = _array[3 .. 5]; 520 | result.startCellContantArea = _array[5 .. 7]; 521 | result.fragmentedFreeBytes = _array[7]; 522 | if (result.isInteriorPage) { 523 | result._rightmostPointer = _array[8 .. 12]; 524 | } 525 | 526 | return result; 527 | } else { 528 | return *(cast (BTreePageHeader*) _array.ptr); 529 | } 530 | } 531 | 532 | bool isInteriorPage() pure const { 533 | return (pageType == pageType.indexInteriorPage 534 | || pageType == pageType.tableInteriorPage); 535 | } 536 | 537 | @property auto pageType() const pure { 538 | return cast(const) _pageType; 539 | } 540 | 541 | @property BigEndian!uint rightmostPointer() { 542 | assert(isInteriorPage, 543 | "the rightmost pointer is only in interior nodes"); 544 | return _rightmostPointer; 545 | } 546 | 547 | @property void rightmostPointer(uint rmp) { 548 | assert(isInteriorPage, 549 | "the rightmost pointer is only in interior nodes"); 550 | _rightmostPointer = rmp; 551 | } 552 | 553 | uint length() pure const { 554 | return 12 - (isInteriorPage ? 0 : 4); 555 | } 556 | 557 | // string toString() { 558 | // import std.conv; 559 | // 560 | // string result = "pageType:\t"; 561 | // result ~= to!string(pageType) ~ "\n"; 562 | // result ~= "firstFreeBlock:\t"; 563 | // result ~= to!string(firstFreeBlock) ~ "\n"; 564 | // result ~= "cellsInPage:\t"; 565 | // result ~= to!string(cellsInPage) ~ "\n"; 566 | // result ~= "startCellContantArea:\t"; 567 | // result ~= to!string(startCellContantArea) ~ "\n"; 568 | // result ~= "fragmentedFreeBytes:\t"; 569 | // result ~= to!string(fragmentedFreeBytes) ~ "\n"; 570 | // if (isInteriorPage) { 571 | // result ~= "_rightmostPointer"; 572 | // result ~= to!string(_rightmostPointer) ~ "\n"; 573 | // } 574 | // 575 | // return result; 576 | // } 577 | } 578 | 579 | bool hasPayload() const pure { 580 | final switch (pageType) with (BTreePageType) { 581 | case emptyPage: 582 | return false; 583 | case tableInteriorPage: 584 | return false; 585 | case indexInteriorPage: 586 | return true; 587 | case indexLeafPage: 588 | return true; 589 | case tableLeafPage: 590 | return true; 591 | } 592 | } 593 | 594 | auto payloadSize(BigEndian!ushort cp) { 595 | assert(hasPayload); 596 | final switch (pageType) with (BTreePageType) { 597 | case emptyPage: 598 | case tableInteriorPage: 599 | assert(0, "page has no payload"); 600 | 601 | case indexInteriorPage: 602 | return VarInt(page[cp + uint.sizeof .. $]); 603 | case indexLeafPage: 604 | return VarInt(page[cp .. $]); 605 | case tableLeafPage: 606 | return VarInt(page[cp .. $]); 607 | } 608 | } 609 | 610 | BTreePageHeader header() const pure { 611 | return BTreePageHeader.fromArray(page[headerOffset.. headerOffset + BTreePageHeader.sizeof]); 612 | } 613 | 614 | BigEndian!ushort[] getCellPointerArray() const pure { 615 | auto offset = header.length + headerOffset; 616 | return page[offset .. offset + header.cellsInPage * ushort.sizeof] 617 | .toArray!(BigEndian!ushort)(header.cellsInPage); 618 | } 619 | 620 | BTreePageType pageType() const pure { 621 | return (header()).pageType; 622 | } 623 | 624 | struct PayloadHeader { 625 | const ubyte[] payloadHeader; 626 | uint offset; 627 | uint _length; 628 | 629 | void popFront() pure { 630 | assert(offset < payloadHeader.length); 631 | offset += _length; 632 | _length = 0; 633 | } 634 | 635 | Payload.SerialTypeCode front() pure { 636 | auto v = VarInt(payloadHeader[offset .. $]); 637 | _length = cast(uint)v.length; 638 | return Payload.SerialTypeCode(v); 639 | } 640 | 641 | bool empty() pure const { 642 | return offset == payloadHeader.length; 643 | } 644 | } 645 | 646 | 647 | } 648 | } 649 | 650 | struct OverflowInfo { 651 | const(ubyte)[] pageSlice; 652 | uint nextPageIdx; 653 | 654 | this(const ubyte[] payloadStart, int offset, const uint payloadSize, const Database.PageRange pages, const uint payloadHeaderSize, const Database.BTreePageType pageType) pure { 655 | import std.algorithm : min; 656 | 657 | uint x; 658 | if (isIndex(pageType)) { 659 | x = ((pages.usablePageSize - 12) * 32 / 255) - 23; 660 | } else { 661 | x = pages.usablePageSize - 35; 662 | } 663 | 664 | if (payloadSize > x) { 665 | auto m = ((pages.usablePageSize - 12) * 32 / 255) - 23; 666 | auto k = m + ((payloadSize - m) % (pages.usablePageSize - 4)); 667 | 668 | auto payloadOnFirstPage = (k <= x ? k : m) - payloadHeaderSize; 669 | 670 | nextPageIdx = BigEndian!uint(payloadStart[payloadOnFirstPage .. payloadOnFirstPage + uint.sizeof]); 671 | 672 | if(offset > payloadOnFirstPage) { 673 | offset -= payloadOnFirstPage; 674 | gotoNextPage(pages); 675 | 676 | auto payloadOnOverflowPage = pages.usablePageSize - uint.sizeof; 677 | while(offset>payloadOnOverflowPage) { 678 | gotoNextPage(pages); 679 | offset -= payloadOnOverflowPage; 680 | } 681 | } else { 682 | pageSlice = payloadStart[0 .. payloadOnFirstPage]; 683 | } 684 | } else { 685 | pageSlice = payloadStart; 686 | } 687 | pageSlice = pageSlice[offset .. $]; 688 | } 689 | 690 | void gotoNextPage(const Database.PageRange pages) pure { 691 | assert(nextPageIdx != 0, "No next Page to go to"); 692 | auto nextPage = pages[nextPageIdx - 1]; 693 | nextPageIdx = BigEndian!uint(nextPage.page[0 .. uint.sizeof]); 694 | pageSlice = nextPage.page[uint.sizeof .. $]; 695 | } 696 | 697 | } 698 | 699 | 700 | static Database.Payload extractPayload( 701 | OverflowInfo* overflowInfo, 702 | const Database.Payload.SerialTypeCode typeCode, 703 | const Database.PageRange pages, 704 | ) pure { 705 | 706 | if (overflowInfo.pageSlice.length >= typeCode.length) { 707 | auto _length = cast(uint) typeCode.length; 708 | 709 | auto payload = overflowInfo.pageSlice[0 .. _length]; 710 | overflowInfo.pageSlice = overflowInfo.pageSlice[_length .. $]; 711 | 712 | if (overflowInfo.pageSlice.length == uint.sizeof) { 713 | assert(0, "I do not expect us to ever get here\n" ~ 714 | "If we ever do, uncomment the two lines below and delete this assert"); 715 | // overflowInfo.nextPageIdx = BigEndian(overflowInfo.pageSlice[0 .. uint.sizeof]); 716 | // overflowInfo.gotoNextPage(pages); 717 | } 718 | 719 | return extractPayload(payload, typeCode); 720 | } else { // typeCode.length > payloadOnFirstPage 721 | alias et = Database.Payload.SerialTypeCode.SerialTypeCodeEnum; 722 | // let's assume SQLite is sane and does not split primitive types in the middle 723 | assert(typeCode.type == et.blob || typeCode.type == et._string); 724 | 725 | auto remainingBytesOfPayload = cast(uint) typeCode.length; 726 | ubyte[] _payloadBuffer; 727 | 728 | for (;;) { 729 | 730 | import std.algorithm : min; 731 | 732 | auto readBytes = cast(uint) min(overflowInfo.pageSlice.length, 733 | remainingBytesOfPayload); 734 | 735 | remainingBytesOfPayload -= readBytes; 736 | 737 | _payloadBuffer ~= overflowInfo.pageSlice[0 .. readBytes]; 738 | 739 | if (remainingBytesOfPayload == 0) { 740 | assert(typeCode.length == _payloadBuffer.length); 741 | 742 | return extractPayload(cast(const)_payloadBuffer, typeCode); 743 | } else { 744 | if (overflowInfo.nextPageIdx == 0 && !remainingBytesOfPayload) { 745 | //This is a very special-case 746 | //Hopefully we don't hit it :) 747 | assert(0, "Moow!"); 748 | } 749 | overflowInfo.gotoNextPage(pages); 750 | } 751 | } 752 | } 753 | } 754 | 755 | static Database.Payload extractPayload(const ubyte[] startPayload, 756 | const Database.Payload.SerialTypeCode typeCode) pure { 757 | Database.Payload p; 758 | p.typeCode = typeCode; 759 | 760 | final switch (typeCode.type) { 761 | case typeof(typeCode).int8: 762 | p.int8 = *cast(byte*) startPayload; 763 | break; 764 | case typeof(typeCode).int16: 765 | p.int16 = *cast(BigEndian!short*) startPayload; 766 | break; 767 | case typeof(typeCode).int24: 768 | p.int24 = (*cast(BigEndian!int*) startPayload) & 0xfff0; 769 | break; 770 | case typeof(typeCode).int32: 771 | p.int32 = *cast(BigEndian!int*) startPayload; 772 | break; 773 | case typeof(typeCode).int48: 774 | p.int48 = (*cast(BigEndian!long*) startPayload) & 0xffffff00; 775 | break; 776 | case typeof(typeCode).int64: 777 | p.int64 = *cast(BigEndian!long*) startPayload; 778 | break; 779 | case typeof(typeCode).float64: 780 | if (!__ctfe) { 781 | p.float64 = *cast(BigEndian!double*) startPayload; 782 | assert(0, "Not supported at runtime either it's BigEndian :)"); 783 | } else 784 | assert(0, "not supporeted at CTFE yet"); 785 | // break; 786 | case typeof(typeCode).blob: 787 | p.blob = cast(ubyte[]) startPayload[0 .. cast(uint) typeCode.length]; 788 | break; 789 | case typeof(typeCode)._string: 790 | p._string = cast(string) startPayload[0 .. cast(uint) typeCode.length]; 791 | break; 792 | 793 | case typeof(typeCode).NULL: 794 | case typeof(typeCode).bool_false: 795 | case typeof(typeCode).bool_true: 796 | break; 797 | } 798 | 799 | return p; 800 | } 801 | 802 | 803 | 804 | auto apply(alias handler)(const Database.Payload p) { 805 | final switch (p.typeCode.type) with (typeof(p.typeCode.type)) { 806 | case NULL: 807 | static if (__traits(compiles, handler(null))) { 808 | return handler(null); 809 | } else { 810 | assert(0, "handler cannot take null"); 811 | } 812 | case int8: 813 | static if (__traits(compiles, handler(p.int8))) { 814 | return handler(p.int8); 815 | } else { 816 | assert(0); 817 | } 818 | case int16: 819 | static if (__traits(compiles, handler(p.int16))) { 820 | return handler(p.int16); 821 | } else { 822 | assert(0); 823 | } 824 | case int24: 825 | static if (__traits(compiles, handler(p.int24))) { 826 | return handler(p.int24); 827 | } else { 828 | assert(0); 829 | } 830 | case int32: 831 | static if (__traits(compiles, handler(p.int32))) { 832 | return handler(p.int32); 833 | } else { 834 | assert(0); 835 | } 836 | case int48: 837 | static if (__traits(compiles, handler(p.int48))) { 838 | return handler(p.int48); 839 | } else { 840 | assert(0); 841 | } 842 | case int64: 843 | static if (__traits(compiles, handler(p.int64))) { 844 | return handler(p.int64); 845 | } else { 846 | assert(0); 847 | } 848 | case float64: 849 | static if (__traits(compiles, handler(p.float64))) { 850 | return handler(p.float64); 851 | } else { 852 | assert(0); 853 | } 854 | case bool_false: 855 | static if (__traits(compiles, handler(false))) { 856 | return handler(false); 857 | } else { 858 | assert(0); 859 | } 860 | case bool_true: 861 | static if (__traits(compiles, handler(true))) { 862 | return handler(true); 863 | } else { 864 | assert(0); 865 | } 866 | case blob: 867 | static if (__traits(compiles, handler(p.blob))) { 868 | return handler(p.blob); 869 | } else { 870 | assert(0); 871 | } 872 | case _string: 873 | static if (__traits(compiles, handler(p._string))) { 874 | return handler(p._string); 875 | } else { 876 | assert(0,"handler " ~ typeof(handler).stringof ~ " cannot be called with string"); 877 | } 878 | } 879 | } 880 | pure : 881 | auto getAs(T)(Database.Payload p) { 882 | return p.apply!(v => cast(T) v); 883 | } 884 | 885 | auto getAs(T)(Database.Row r, uint columnIndex) { 886 | return r.column(columnIndex).getAs!T(); 887 | } 888 | 889 | auto getAs(T)(Database.Row r, Database.TableSchema s, string colName) { 890 | return r.getAs!T(s.getcolumnIndex(colName)); 891 | } 892 | 893 | unittest { 894 | Database.Payload p; 895 | p.typeCode.type = Database.Payload.SerialTypeCodeEnum.bool_true; 896 | assert(p.getAs!(int) == 1); 897 | p.typeCode.type = Database.Payload.SerialTypeCodeEnum.bool_false; 898 | assert(p.getAs!(int) == 0); 899 | } 900 | 901 | bool isIndex(const Database.BTreePage.BTreePageType pageType) pure { 902 | return ((pageType & 2) ^ 2) == 0; 903 | } 904 | 905 | static assert(isIndex(Database.BTreePageType.indexLeafPage) && isIndex(Database.BTreePageType.indexInteriorPage) && !isIndex(Database.BTreePageType.tableLeafPage)); 906 | -------------------------------------------------------------------------------- /source/sqlstuff.d: -------------------------------------------------------------------------------- 1 | //sqlstuff 2 | 3 | struct TableInfo { 4 | ColumInfo[] columInfos; 5 | string tableName; 6 | 7 | this(string tableName, ColumInfo[] columInfos) pure { 8 | this.tableName = tableName; 9 | this.columInfos = columInfos; 10 | } 11 | } 12 | 13 | struct ColumInfo { 14 | string name; 15 | string typeName; 16 | bool primaryKey; 17 | bool notNull; 18 | bool unique; 19 | bool autoincrement; 20 | } 21 | -------------------------------------------------------------------------------- /source/test.d: -------------------------------------------------------------------------------- 1 | module test; 2 | 3 | // Copyright Stefan Koch 2015 - 2018. 4 | // Distributed under the Boost Software License, Version 1.0. 5 | // (See accompanying file LICENSE.md or copy at 6 | // http://www.boost.org/LICENSE_1_0.txt) 7 | 8 | import sqlited; 9 | import misc; 10 | import sqlstuff; 11 | 12 | static immutable long_create_table = 13 | q{CREATE TABLE `veryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryverylooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongtttttttttaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbllllllllllleeeeeeeeeeeeeeeeennnnnnaaaaaaaaaaameeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee` ( 14 | `vvvvvvvvvveeeeeeeerrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrryyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyylllllllloooooooooooooooooooooooooooonnnnnnnnnnnnnnnngggggggggggggggggggvvvvvvvvvvvvvvvvffffffffffffffffffffiiiiiiiiiiieeeeeeeeeeeeeeeeellllllllllllllddddddddddddddddddddddnnnnnnnnnnnaaaaaaaaaaaaammmmmmmmmmmmmmmmeeeeeeeeeeeeeeeeee` INTEGER, 15 | `Field2` INTEGER 16 | )}; 17 | static immutable test_s3db = cast(immutable)Database(cast(immutable ubyte[]) import("test.s3db")); 18 | static immutable Database.MasterTableSchema[] schemas = readRows!(r => r.deserialize!(Database.MasterTableSchema))(test_s3db.rootPage, test_s3db.pages); 19 | static assert(schemas[2].sql==long_create_table); 20 | 21 | import frail_sql_parser; 22 | 23 | static assert (parseCreateTable( 24 | q{CREATE TABLE spatial_ref_sys ( 25 | srid INTEGER NOT NULL PRIMARY KEY, 26 | auth_name VARCHAR(256) NOT NULL, 27 | auth_srid INTEGER NOT NULL, 28 | ref_sys_name VARCHAR(256), 29 | proj4text VARCHAR(2048) NOT NULL)} 30 | ) == TableInfo("spatial_ref_sys", [ 31 | ColumInfo("srid", "INTEGER", true, true), 32 | ColumInfo("auth_name", "VARCHAR(256)", false, true), 33 | ColumInfo("auth_srid", "INTEGER", false, true), 34 | ColumInfo("ref_sys_name","VARCHAR(256)",false), 35 | ColumInfo("proj4text","VARCHAR(2048)", false, true), 36 | ]) 37 | ); 38 | static assert(parseCreateTable(long_create_table) == TableInfo("veryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryveryverylooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongtttttttttaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbllllllllllleeeeeeeeeeeeeeeeennnnnnaaaaaaaaaaameeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", [ColumInfo("vvvvvvvvvveeeeeeeerrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrryyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyylllllllloooooooooooooooooooooooooooonnnnnnnnnnnnnnnngggggggggggggggggggvvvvvvvvvvvvvvvvffffffffffffffffffffiiiiiiiiiiieeeeeeeeeeeeeeeeellllllllllllllddddddddddddddddddddddnnnnnnnnnnnaaaaaaaaaaaaammmmmmmmmmmmmmmmeeeeeeeeeeeeeeeeee", "INTEGER"), ColumInfo("Field2", "INTEGER")])); 39 | 40 | static assert(parseCreateTable( 41 | q{CREATE TABLE Towns ( 42 | PK_UID INTEGER PRIMARY KEY AUTOINCREMENT, 43 | Name TEXT, 44 | Peoples INTEGER, 45 | LocalCounc INTEGER, 46 | County INTEGER, 47 | Region INTEGER, "Geometry" POINT) 48 | } 49 | ) == TableInfo("Towns", [ 50 | ColumInfo("PK_UID", "INTEGER", true, false, false, true), 51 | ColumInfo("Name", "TEXT"), 52 | ColumInfo("Peoples", "INTEGER"), 53 | ColumInfo("LocalCounc", "INTEGER"), 54 | ColumInfo("County", "INTEGER"), 55 | ColumInfo("Region", "INTEGER"), 56 | ColumInfo("Geometry", "POINT") 57 | ]) 58 | ); 59 | -------------------------------------------------------------------------------- /source/utils.d: -------------------------------------------------------------------------------- 1 | module utils; 2 | /*************************** 3 | * Utils used by SQLite-D * 4 | * By Stefan Koch 2016 * 5 | ***************************/ 6 | 7 | // Copyright Stefan Koch 2015 - 2018. 8 | // Distributed under the Boost Software License, Version 1.0. 9 | // (See accompanying file LICENSE.md or copy at 10 | // http://www.boost.org/LICENSE_1_0.txt) 11 | 12 | 13 | /** this struct tries to keep it's own value as BigEndian */ 14 | struct BigEndian(T) { 15 | T asNative; 16 | alias asBigEndian this; 17 | pure nothrow @safe @nogc : 18 | @property T asBigEndian() { 19 | return swapIfNeeded(asNative); 20 | } 21 | 22 | @property asBigEndian(U)(U val) if(is(U == T)) { 23 | return asNative = swapIfNeeded(val); 24 | } 25 | 26 | alias isBigEndian = void; 27 | 28 | this (T val) { 29 | static if (is(T.isBigEndian)) { 30 | this.asNative = val.asNative; 31 | } else { 32 | this.asNative = swapIfNeeded(val); 33 | } 34 | } 35 | 36 | this(const ubyte[] _array) @trusted { 37 | assert(T.sizeof == _array.length); 38 | foreach(i;0 .. T.sizeof) { 39 | asNative |= (_array[i] << i*8UL); 40 | } 41 | } 42 | 43 | BigEndian!T opAssign(BigEndian!T val) { 44 | this.asNative = val.asNative; 45 | return this; 46 | } 47 | 48 | import std.traits : isIntegral; 49 | BigEndian!T opAssign(U)(U val) if(!is(U.isBigEndian) && isIntegral!U) { 50 | assert(val <= T.max && val >= T.min); 51 | this.asNative = swapIfNeeded(cast(T)val); 52 | return this; 53 | } 54 | 55 | BigEndian!T opAssign(U)(U val) if(is(U : const ubyte[])) { 56 | this = BigEndian!T(val); 57 | return this; 58 | } 59 | 60 | static U swapIfNeeded (U)(U val) { 61 | import std.bitmanip:swapEndian; 62 | 63 | version(BigEndian) { 64 | return val; 65 | } else { 66 | static if (is(U.isBigEndian)) { 67 | return val; 68 | } else { 69 | enum _2066_cannot_handle_swapEndian = false; 70 | static if (_2066_cannot_handle_swapEndian) { 71 | static if (U.sizeof == 8) { 72 | return (((val & 0x00000000000000ffUL) << 56UL) | 73 | ((val & 0x000000000000ff00UL) << 40UL) | 74 | ((val & 0x0000000000ff0000UL) << 24UL) | 75 | ((val & 0x00000000ff000000UL) << 8UL) | 76 | ((val & 0x000000ff00000000UL) >> 8UL) | 77 | ((val & 0x0000ff0000000000UL) >> 24UL) | 78 | ((val & 0x00ff000000000000UL) >> 40UL) | 79 | ((val & 0xff00000000000000UL) >> 56UL) 80 | ); 81 | } else static if (U.sizeof == 4) { 82 | return ((val & 0x000000ff) << 24) | 83 | ((val & 0x0000ff00) << 8) | 84 | ((val & 0x00ff0000) >> 8) | 85 | ((val & 0xff000000) >> 24); 86 | } else static if (U.sizeof == 2) { 87 | return cast(ushort)(((val & 0xff00) >> 8) | 88 | ((val & 0x00ff) << 8)); 89 | } else static if (U.sizeof == 1) { 90 | assert(0, "you should not use BigEndian for byte-sized vaules"); 91 | } else { 92 | assert(0, "cannot swap this byteSize"); 93 | } 94 | } else { 95 | return swapEndian(val); 96 | } 97 | } 98 | } 99 | } 100 | 101 | version(BigEndian) {} else { 102 | static assert(swapIfNeeded(0xABCD_EF01_2345_6789) == 0x8967_4523_01EF_CDAB); 103 | static assert(swapIfNeeded(0x0123_4567_89AB_CDEF) == 0xEFCD_AB89_6745_2301); 104 | } 105 | } 106 | 107 | auto bigEndian(T)(T val) pure { 108 | static if (is(T.isBigEndian)) { 109 | return BigEndian!(typeof(val.asNative))(val.asNative); 110 | } else { 111 | return BigEndian!T(val); 112 | } 113 | } 114 | 115 | T[] toArray(T)(const ubyte[] _array, const size_t size) { 116 | if (__ctfe) { 117 | T[] result; 118 | alias sliceType = typeof(_array[0 .. T.sizeof]); 119 | 120 | result.length = size; 121 | 122 | foreach(i; 0 .. size) { 123 | const pos = i * T.sizeof; 124 | static if (is(typeof(T(sliceType.init)))) { 125 | result[i] = T(_array[pos .. pos + T.sizeof]); 126 | } else { 127 | static assert(0, T.stringof ~ " has to have a constructor taking " ~ sliceType.stringof); 128 | } 129 | } 130 | 131 | return result; 132 | } else { 133 | return cast(T[])(_array); 134 | } 135 | } 136 | 137 | T fromArray(T)(const ubyte[] _array) { 138 | if (__ctfe) { 139 | uint offset; 140 | T result; 141 | static assert(T.alignof == 1, "Be sure to use this only on align(1) structures!"); 142 | assert(_array.length >= T.sizeof, "your input array needs to be at least as long as your type.sizeof"); 143 | 144 | ///XXX this cucially depends on your type being byte aligned! 145 | foreach (member; __traits(derivedMembers, struct_type)) { 146 | alias type = typeof(__traits(getMember, instance, member)); 147 | 148 | static if (!(is(type == function) || is(type == const))) { 149 | alias sliceType = typeof(_array[0 .. type.sizeof]); 150 | static if (is(typeof(type(sliceType.init)))) { 151 | __traits(getMember, result, member) = type(_array[offset .. offset + type.sizeof]); 152 | } else static if (type.sizeof == sliceType.init[0].sizeof && is(typeof(cast(type)(sliceType.init[0])))) { 153 | __traits(getMember, result, member) = type(_array[offset .. offset + type.sizeof][0]); 154 | } else { 155 | static assert(0, T.stringof ~ " has to have a constructor taking or needs to be castable to ubyte" ~ sliceType.stringof); 156 | } 157 | offset += type.sizeof; 158 | assert(__traits(getMember, result, member).alignof == offset); 159 | } 160 | } 161 | 162 | return result; 163 | } else { 164 | return *(cast(T*) _array.ptr); 165 | } 166 | } 167 | 168 | uint sizeInBytes(ulong val) pure @nogc nothrow { 169 | foreach(n;0 .. cast(uint)ulong.sizeof) { 170 | if (!(val >>= 8)) { 171 | return n+1; 172 | } else { 173 | continue; 174 | } 175 | } 176 | assert(0); 177 | } 178 | import std.range : isRandomAccessRange, ElementType; 179 | 180 | struct SkipArray(T) if (isRandomAccessRange!(ElementType!T)) { 181 | const(ElementType!T)[] arrays; 182 | size_t _length; 183 | 184 | @property const(size_t) length() const pure { 185 | return cast(const) _length; 186 | } 187 | 188 | auto opOpAssign (string op)(const T rhs) { 189 | static if (op == "~") { 190 | arrays ~= rhs; 191 | _length += rhs.length; 192 | } else { 193 | assert(0, "Operator " ~ op ~ " not supported"); 194 | } 195 | } 196 | 197 | const auto opIndex(const size_t idx) { 198 | assert(idx < length); 199 | size_t currentPos; 200 | foreach(ref a;arrays) { 201 | if (idx >= a.length + currentPos) { 202 | currentPos += a.length; 203 | } else { 204 | return a[idx - currentPos]; 205 | } 206 | } 207 | assert(0, "invalid idx"); 208 | } 209 | 210 | this(T arrays) { 211 | this.arrays = arrays; 212 | foreach(a;arrays) { 213 | _length += a.length; 214 | } 215 | } 216 | } 217 | 218 | auto skipArray(T)(T t) { 219 | return SkipArray!T(t); 220 | } 221 | 222 | static immutable intArrArr = skipArray([[1,2],[3],[4,5,6],[7,8,9]]); 223 | static assert(intArrArr.length == 9); 224 | static assert(intArrArr[3] == 4); 225 | static assert(intArrArr[8] == 9); 226 | static assert(intArrArr[0] == 1); 227 | 228 | unittest { 229 | auto arr = skipArray([["Hello"]]); 230 | arr ~= [["beautiful"], ["world"]]; 231 | assert(arr.length == 3); 232 | } 233 | 234 | //TODO implement this! 235 | double float64(const ubyte[] _bytes) { 236 | assert(_bytes.length > double.sizeof); 237 | enum bias = 1023; 238 | enum mantissa_length = 53; 239 | enum exponent_length = 11; 240 | assert(mantissa_length + exponent_length == 64); 241 | double result; 242 | 243 | foreach(i; 0 .. (mantissa_length / 8) + 1) { 244 | 245 | } 246 | 247 | return result; 248 | } 249 | -------------------------------------------------------------------------------- /source/varint.d: -------------------------------------------------------------------------------- 1 | module varint; 2 | import utils; 3 | 4 | // Copyright Stefan Koch 2015 - 2018. 5 | // Distributed under the Boost Software License, Version 1.0. 6 | // (See accompanying file LICENSE.md or copy at 7 | // http://www.boost.org/LICENSE_1_0.txt) 8 | 9 | enum unrolled = true; 10 | 11 | struct VarInt 12 | { 13 | pure nothrow @safe @nogc: 14 | this(BigEndian!long value) @trusted 15 | { 16 | //FIXME the constructor does not work for value bigger then uint.max; 17 | auto len = lengthInVarInt(value); 18 | long beValue = value.asBigEndian; 19 | 20 | auto _len = len; 21 | while (_len--) 22 | { 23 | _storage.ptr[_len] = (beValue & 0x7f) | 0x80; 24 | beValue >>= 7; 25 | } 26 | _storage.ptr[len - 1] &= 0x7f; 27 | 28 | byteArray = _storage[0 .. len]; 29 | } 30 | 31 | const ubyte[] byteArray; 32 | ubyte[9] _storage; 33 | alias toBeLong this; 34 | 35 | //TODO FIXME toBeLong does not correctly convert negative Numbers 36 | 37 | @property BigEndian!long toBeLong() @trusted 38 | { 39 | long tmp; 40 | static if (unrolled) 41 | { 42 | uint v3 = 0; 43 | 44 | if (byteArray.ptr[0] & 0x80) 45 | { 46 | v3 = 1; 47 | if (byteArray.ptr[1] & 0x80) 48 | { 49 | v3 = 2; 50 | if (byteArray.ptr[2] & 0x80) 51 | { 52 | v3 = 3; 53 | if (byteArray.ptr[3] & 0x80) 54 | { 55 | v3 = 4; 56 | if (byteArray.ptr[4] & 0x80) 57 | { 58 | v3 = 5; 59 | if (byteArray.ptr[5] & 0x80) 60 | { 61 | v3 = 6; 62 | if (byteArray.ptr[6] & 0x80) 63 | { 64 | v3 = 7; 65 | if (byteArray.ptr[7] & 0x80) 66 | v3 = 8; 67 | } 68 | } 69 | } 70 | } 71 | } 72 | } 73 | } 74 | else 75 | { 76 | BigEndian!long result; 77 | 78 | version (LittleEndian) 79 | { 80 | result.asNative = (cast(long) byteArray.ptr[0] << 56); 81 | } 82 | else 83 | { 84 | result.asNative = (cast(long) byteArray.ptr[0]); 85 | } 86 | return result; 87 | } 88 | 89 | } 90 | else 91 | { 92 | uint length = length(); 93 | } 94 | 95 | BigEndian!long result; 96 | 97 | // The hottest loop in the whole program! 98 | //TODO There must be a way to speed it up! 99 | static if (unrolled) 100 | { 101 | switch (v3) 102 | { 103 | case 8: 104 | tmp |= ((cast(long) byteArray.ptr[8]) << 7 * (v3 - 8)); 105 | goto case 7; 106 | case 7: 107 | tmp |= ((cast(long) byteArray.ptr[7] & 0x7FUL) << 7 * (v3 - 7)); 108 | goto case 6; 109 | case 6: 110 | tmp |= ((cast(long) byteArray.ptr[6] & 0x7FUL) << 7 * (v3 - 6)); 111 | goto case 5; 112 | case 5: 113 | tmp |= ((cast(long) byteArray.ptr[5] & 0x7FUL) << 7 * (v3 - 5)); 114 | goto case 4; 115 | case 4: 116 | tmp |= ((cast(long) byteArray.ptr[4] & 0x7FUL) << 7 * (v3 - 4)); 117 | goto case 3; 118 | case 3: 119 | tmp |= ((cast(long) byteArray.ptr[3] & 0x7FUL) << 7 * (v3 - 3)); 120 | goto case 2; 121 | case 2: 122 | tmp |= ((cast(long) byteArray.ptr[2] & 0x7FUL) << 7 * (v3 - 2)); 123 | goto case 1; 124 | case 1: 125 | tmp |= ((cast(long) byteArray.ptr[1] & 0x7FUL) << 7 * (v3 - 1)); 126 | tmp |= ((cast(long) byteArray.ptr[0] & 0x7FUL) << 7 * (v3 - 0)); 127 | break; 128 | default: 129 | assert(0); 130 | 131 | } 132 | } 133 | else 134 | 135 | { 136 | foreach (idx; 0 .. length) 137 | { 138 | ubyte val = byteArray[idx]; 139 | long maskedVal = (cast(long) val & 0x7fUL); // mask 8th bit 140 | long shiftBy = (length - idx - 1UL) * 7UL; 141 | if (idx < 8) 142 | { 143 | tmp |= (maskedVal << shiftBy); 144 | } 145 | else 146 | { 147 | tmp |= (cast(long) val << 63UL); 148 | } 149 | } 150 | } 151 | //this evokes swapIfNeeded 152 | result = tmp; 153 | 154 | return result; 155 | } 156 | 157 | this(const ubyte[] _arr) 158 | { 159 | this.byteArray = _arr; 160 | } 161 | 162 | static int lengthInVarInt(BigEndian!long value) 163 | { 164 | if (value > 1L << 56 || value < 0) 165 | { 166 | return 9; 167 | } 168 | else if (value < 1 << 7) 169 | { 170 | return 1; 171 | } 172 | else if (value < 1 << 14) 173 | { 174 | return 2; 175 | } 176 | else if (value < 1 << 21) 177 | { 178 | return 3; 179 | } 180 | else if (value < 1 << 28) 181 | { 182 | return 4; 183 | } 184 | else if (value < 1L << 35) 185 | { 186 | return 5; 187 | } 188 | else if (value < 1L << 42) 189 | { 190 | return 6; 191 | } 192 | else if (value < 1L << 49) 193 | { 194 | return 7; 195 | } 196 | else if (value < 1L << 56) 197 | { 198 | return 8; 199 | } 200 | assert(0, "We should never get here"); 201 | } 202 | 203 | @property uint length() 204 | { 205 | return _length(byteArray); 206 | } 207 | 208 | static uint _length(const ubyte[] arr) 209 | { 210 | 211 | foreach (idx; 0 .. 9) 212 | { 213 | if (arr[idx] & (1 << 7)) 214 | { 215 | continue; 216 | } 217 | else 218 | { 219 | return idx + 1; 220 | } 221 | assert(0, "we should never get here"); 222 | } 223 | return 9; 224 | } 225 | 226 | static assert(_length((cast(ubyte[])[0x6d, 0x00])) == 1); 227 | static assert(_length((cast(ubyte[])[0x7f, 0x00])) == 1); 228 | static assert(_length((cast(ubyte[])[0x82, 0x12])) == 2); 229 | static assert(_length((cast(ubyte[])[0xfe, 0xfe, 0x00])) == 3); 230 | static assert(VarInt((cast(ubyte[])[0x81, 0x01])).toBeLong == 129); 231 | static assert(VarInt((cast(ubyte[])[0x81, 0x00])).toBeLong == 0x0080); 232 | static assert(VarInt((cast(ubyte[])[0x82, 0x00])).toBeLong == 0x0100); // should be 0x0100 233 | static assert(_length((cast(ubyte[])[0x82, 0x80, 0x00])) == 3); 234 | static assert(VarInt((cast(ubyte[])[0x84, 0x60, 0x00])).toBeLong == 608); 235 | //FIXME make this work! 236 | // static assert(VarInt(cast(ubyte[])[0xFF, 0xFF, 0xFF, 0XFF, 0xFF, 0XFF, 0xFF, 0xFF, 0xEA]).toBeLong == -22); 237 | static assert(VarInt(bigEndian!long(265)).toBeLong == 265); 238 | static assert(VarInt(bigEndian!long(6421)).toBeLong == 6421); 239 | static assert(VarInt(bigEndian!long(22)).toBeLong == 22); 240 | static assert(VarInt.lengthInVarInt(BigEndian!long(-22)) == 9); 241 | static assert(VarInt(bigEndian!long(uint.max)).toBeLong == uint.max); 242 | //static assert(VarInt(cast(ubyte[])[0xFF, 0xFF, 0xFF, 0XFF, 0xFF, 0XFF, 243 | // 0xFF, 0xFF, 0xEA]) == VarInt(bigEndian(-22L))); 244 | //static assert (VarInt().lengthInVarInt(608) == 2); 245 | static assert(VarInt((cast(ubyte[])[0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 246 | 0x89])).toBeLong != 0); 247 | } 248 | -------------------------------------------------------------------------------- /views/test-2.3.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/UplinkCoder/sqlite-d/1ee3450996bddf257b8451ea1b7663811a222c75/views/test-2.3.sqlite -------------------------------------------------------------------------------- /views/test.s3db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/UplinkCoder/sqlite-d/1ee3450996bddf257b8451ea1b7663811a222c75/views/test.s3db -------------------------------------------------------------------------------- /views/test2.s3db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/UplinkCoder/sqlite-d/1ee3450996bddf257b8451ea1b7663811a222c75/views/test2.s3db -------------------------------------------------------------------------------- /views/test3.s3db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/UplinkCoder/sqlite-d/1ee3450996bddf257b8451ea1b7663811a222c75/views/test3.s3db -------------------------------------------------------------------------------- /views/test4.s3db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/UplinkCoder/sqlite-d/1ee3450996bddf257b8451ea1b7663811a222c75/views/test4.s3db -------------------------------------------------------------------------------- /views/test_chaged.s3db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/UplinkCoder/sqlite-d/1ee3450996bddf257b8451ea1b7663811a222c75/views/test_chaged.s3db --------------------------------------------------------------------------------