├── dumplite ├── ListRange.pyc ├── SQLiteFormat.pyc ├── personas.sqlite ├── ListRange.py ├── dumplite.py └── SQLiteFormat.py ├── README.md └── recoversqlite.py /dumplite/ListRange.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aramosf/recoversqlite/HEAD/dumplite/ListRange.pyc -------------------------------------------------------------------------------- /dumplite/SQLiteFormat.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aramosf/recoversqlite/HEAD/dumplite/SQLiteFormat.pyc -------------------------------------------------------------------------------- /dumplite/personas.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aramosf/recoversqlite/HEAD/dumplite/personas.sqlite -------------------------------------------------------------------------------- /dumplite/ListRange.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | class ListRange(list): 3 | def __init__(self, r=[]): 4 | if len(r): 5 | self.addRange( r ) 6 | def addRange(self, r ): 7 | if r[1] <= r[0]: 8 | return self 9 | new = ListRange() 10 | while len(self) > 0: 11 | i = self.pop(0) 12 | if i[0] > r[1]+1: 13 | self.insert(0, i) 14 | break 15 | if r[0] > i[1]+1: 16 | new.append(i) 17 | continue 18 | if i[0] <= r[0]: 19 | r[0] = i[0] 20 | if i[1] >= r[1]: 21 | r[1] = i[1] 22 | break 23 | new.append(r) 24 | new.extend(self) 25 | self[:] = new[:] 26 | return new 27 | 28 | def delRange(self, r): 29 | if r[1] < r[0]: 30 | return self 31 | new = ListRange() 32 | while len(self) > 0: 33 | i = self.pop(0) 34 | if i[0] > r[1]+1: 35 | self.insert(0, i) 36 | break 37 | if r[0] > i[1]+1: 38 | new.append(i) 39 | continue 40 | if i[0] < r[0]: 41 | self.insert(0, [r[0]+1, i[1]]) 42 | new.append([i[0], r[0]-1]) 43 | continue 44 | if i[1] > r[1]: 45 | self.insert(0, [r[1]+1, i[1]]) 46 | break 47 | new.extend(self) 48 | self[:] = new[:] 49 | return new 50 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # recoversqlite 2 | Two different scripts with the same pourpose: recover information from deleted registers in sqlite databases. 3 | - recoversqlite.py - first version 4 | - dumplite.py - rewrited, second version 5 | 6 | ### Version 7 | 1.0 8 | 9 | ### Tech 10 | All of this was part of SecurityByDefault forensics in sqlite articles, you can read more in spanish here: 11 | 12 | * http://www.securitybydefault.com/2012/08/forense-de-sqlite-i-recursos.html 13 | * http://www.securitybydefault.com/2012/08/forense-de-sqlite-ii-cabecera.html 14 | * http://www.securitybydefault.com/2012/08/forense-de-sqlite-iii-paginas-libres.html 15 | * http://www.securitybydefault.com/2012/08/forense-de-sqlite-iv-paginas-b-tree-y.html 16 | * http://www.securitybydefault.com/2012/09/forense-de-sqlite-v-celdas.html 17 | * http://www.securitybydefault.com/2012/09/forense-de-sqlite-vi-practica.html 18 | * http://www.securitybydefault.com/2012/09/forense-de-sqlite-vii-ejercicio-resuelto.html 19 | 20 | 21 | ### Installation 22 | just python: 23 | 24 | ```sh 25 | $ python recoversqlite.py -f personas.sqlite 26 | ``` 27 | 28 | ```sh 29 | $ python dumplite.py -f personas.sqlite -l -u -d 30 | ``` 31 | 32 | ### References 33 | 34 | * http://forensicsfromthesausagefactory.blogspot.com.es/2011/04/carving-sqlite-databases-from.html 35 | * http://forensicsfromthesausagefactory.blogspot.com.es/2011/05/analysis-of-record-structure-within.html 36 | * http://forensicsfromthesausagefactory.blogspot.com.es/2011/05/sqlite-pointer-maps-pages.html 37 | * http://forensicsfromthesausagefactory.blogspot.com.es/2011/07/sqlite-overflow-pages-and-other-loose.html 38 | * http://mobileforensics.wordpress.com/2011/04/30/sqlite-records/ 39 | * http://mobileforensics.wordpress.com/2011/09/17/huffman-coding-in-sqlite-a-primer-for-mobile-forensics/ 40 | * http://www.forensicswiki.org/wiki/Mozilla_Firefox_3_History_File_Format 41 | * http://digitalinvestigation.wordpress.com/2012/05/04/the-forensic-implications-of-sqlites-write-ahead-log/ 42 | * http://sandbox.dfrws.org/2011/fox-it/DFRWS2011_results/Report/Sqlite_carving_extractAndroidData.pdf 43 | * http://computer-forensics.sans.org/summit-archives/2011/2-taking-it-to-the-next-level.pdf 44 | * http://www.ccl-forensics.com/images/f3%20presentation3.pdf 45 | 46 | Official Doc: 47 | * http://www.sqlite.org/fileformat.html 48 | * http://www.sqlite.org/fileformat2.html 49 | * http://sqlite.org/docs.html 50 | * http://www.sqlite.org/pragma.html#pragma_secure_delete 51 | * http://www.sqlite.org/src/artifact/cce1c3360c 52 | 53 | Tools (public and commercial): 54 | * http://www.garykessler.net/software/sqlite_parser_v2.1a.zip 55 | * http://www.crypticbit.com/zen/products/iphoneanalyzer 56 | * http://www.ccl-forensics.com/Software/epilog-from-ccl-forensics.html 57 | * http://www.filesig.co.uk/sqlite-forensic-reporter.html 58 | * http://www.oxygen-forensic.com/en/features/sqliteviewer/ 59 | 60 | Studies and publications: 61 | * http://www.springerlink.com/content/n6l6526n16847kh8/ 62 | * http://www.sciencedirect.com/science/article/pii/S1742287609000048 63 | 64 | Books: 65 | * http://www.amazon.com/Definitive-Guide-SQLite-Mike-Owens/dp/1590596730 66 | * http://books.google.es/books/about/Inside_SQLite.html?id=QoxUx8GOjKMC&redir_esc=y 67 | 68 | -------------------------------------------------------------------------------- /dumplite/dumplite.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python 2 | # -*- encoding: utf-8 -*- 3 | # Marcos Aguero - WiredRat with aramosf :D 4 | 5 | import getopt, sys 6 | import ListRange 7 | from SQLiteFormat import * 8 | from struct import unpack 9 | 10 | def usage(): 11 | print """ 12 | Uso: ./dumplite -f FICHERO [ OPCIONES ] 13 | Parametros: 14 | -f filename Fichero a leer 15 | -F Imprime las cabeceras del fichero 16 | -l Imprime registros en paginas freelist leaf 17 | -h Esta ayuda 18 | -u Imprime los rangos libres 19 | -d Imprime los datos de los rangos libres 20 | """ 21 | 22 | def all_same(items): 23 | return all(x == items[0] for x in items) 24 | 25 | def hexdump(src, rel_pos=0, length=16): 26 | if all_same(src): 27 | strzero = "%04X 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................\n" % rel_pos 28 | strzero += "..." 29 | return strzero 30 | result = [] 31 | digits = 4 if isinstance(src, unicode) else 2 32 | for i in xrange(0, len(src), length): 33 | s = src[i:i+length] 34 | hexa = b' '.join(["%0*X" % (digits, ord(x)) for x in s]) 35 | text = b'' 36 | for x in s: 37 | if(ord(x) < 0x20 or ord(x) == 0x7F or 0x81 <= ord(x) < 0xA0): 38 | text += b'.' 39 | else: 40 | text += x 41 | result.append( b"%04X %-*s |%s|" % (i+rel_pos, length*(digits + 1), 42 | hexa, text) ) 43 | return b'\n'.join(result) 44 | 45 | 46 | def printFreelistLeaf(): 47 | print "\nRecuperando registros de paginas Freelist:" 48 | for i, p in db.Pages.iteritems(): 49 | if p.type == 'Freelist Leaf' and p.isSubType.type != 'Unknown' and p.isSubType.numCells: 50 | print "Página numero: " + str(p.pageNum) 51 | for c in p.isSubType.cells: 52 | payload = db.getCellPayload( c ) 53 | r = getRecord( db.getCellPayload( c ) ) 54 | print ", ".join(["%s:%s" % (f['strType'], f['value']) for f in r]) 55 | print "" 56 | 57 | def printUnallocatedRanges(): 58 | print "\nImprimiendo rangos:" 59 | for i, p in db.Pages.iteritems(): 60 | print str(p.pageNum) + " (" + p.type + "): ", 61 | for r in p.unallocated: 62 | print "[" + str(r[0]+db.pageSize*(p.pageNum-1)) + ", " + str(r[1]+db.pageSize*(p.pageNum-1)) + ']', 63 | print "" 64 | if dump_unallocated: 65 | for r in p.unallocated: 66 | string = p.stream[r[0]:r[1]] 67 | print hexdump(string, db.pageSize*(p.pageNum-1)+r[0]) 68 | if p.isSubType: 69 | print "\t(" + p.isSubType.type + "): ", 70 | for r in p.isSubType.unallocated: 71 | print "[" + str(r[0]+db.pageSize*(p.pageNum-1)) + ", " + str(r[1]+db.pageSize*(p.pageNum-1)) + ']', 72 | print "" 73 | 74 | 75 | 76 | 77 | try: 78 | opts, args = getopt.getopt( sys.argv[1:], 'hf:Flud' ); 79 | except getopt.GetoptError, err: 80 | print str(err) 81 | usage() 82 | sys.exit(1) 83 | 84 | filename = '' 85 | print_file_headers = 0 86 | print_freelist_leaf_records = 0 87 | print_unallocated_ranges = 0 88 | dump_unallocated = 0 89 | 90 | for o, a in opts: 91 | if o == "-h": 92 | usage() 93 | sys.exit(0) 94 | if o == "-f": 95 | filename = a 96 | if o == '-F': 97 | print_file_headers = 1 98 | if o == '-l': 99 | print_freelist_leaf_records = 1 100 | if o == '-u': 101 | print_unallocated_ranges = 1 102 | if o == '-d': 103 | print_unallocated_ranges = 1 104 | dump_unallocated = 1 105 | 106 | 107 | if filename == '': 108 | print "el parametro -f es obligatorio" 109 | sys.exit(2) 110 | 111 | f = open(filename, 'r') 112 | stream = f.read() 113 | db = SQLiteRaw(stream) 114 | 115 | if print_file_headers: 116 | db.dumpHeader() 117 | 118 | if print_freelist_leaf_records: 119 | printFreelistLeaf() 120 | if print_unallocated_ranges: 121 | printUnallocatedRanges(); 122 | -------------------------------------------------------------------------------- /dumplite/SQLiteFormat.py: -------------------------------------------------------------------------------- 1 | from struct import * 2 | from ListRange import * 3 | 4 | def readVarInt(stream, offset=0): 5 | ret = 0 6 | chr = unpack(">B", stream[offset:offset+1] )[0] 7 | of = 1 8 | while(chr & 0x80 and of < 9): 9 | ret = ret << 7 10 | ret += chr & 0x7F 11 | chr = unpack(">B", stream[offset+of:offset+of+1] )[0] 12 | of += 1 13 | ret = ret << 7 14 | ret += chr 15 | return ret, of 16 | 17 | def getRecord( payload ): 18 | (hsize, offset) = readVarInt( payload ) 19 | fields = [] 20 | while offset < hsize: 21 | (ftype, size) = readVarInt( payload, offset ) 22 | fields.append( { 'type': ftype } ) 23 | offset += size 24 | for f in fields: 25 | # Based on http://www.sqlite.org/fileformat2.html#record_format 26 | t = f['type'] 27 | s = '' 28 | if t == 0: 29 | v = 'Nul' 30 | s = 'NULL' 31 | elif t == 1: 32 | v = unpack(">b", payload[offset:offset+1])[0] 33 | s = 'INTEGER(8)' 34 | offset += 1 35 | elif t == 2: 36 | v = unpack( ">h", payload[offset:offset+2] )[0] 37 | s = 'INTEGER(16)' 38 | offset += 2 39 | elif t == 3: 40 | v = 0 41 | for i in unpack( ">3B", payload[offset:offset+3] ): 42 | v = v << 8 43 | v += i 44 | s = 'INTEGER(24)' 45 | offset += 3 46 | elif t == 4: 47 | v = unpack( ">i", payload[offset:offset+4] )[0] 48 | s = 'INTEGER(32)' 49 | offset += 4 50 | elif t == 5: 51 | v = 0 52 | for i in unpack( ">6B", payload[offset:offset+6] ): 53 | v = v << 8 54 | v += i 55 | s = 'INTEGER(48)' 56 | offset += 6 57 | elif t == 6: 58 | v = unpack( ">q", payload[offset:offset+8] )[0] 59 | s = 'INTEGER(64)' 60 | offset += 8 61 | elif t == 7: 62 | v = unpack( ">d", payload[offset:offset+8] )[0] 63 | s = 'FLOAT(64)' 64 | offset += 8 65 | elif t == 8: 66 | v = 0 67 | s = 'CONSTANT' 68 | elif t == 9: 69 | v = 1 70 | s = 'CONSTANT' 71 | elif t == 10 or t == 11: 72 | v = 'Unknown value' 73 | s = 'NOT USED' 74 | elif t > 11 and t%2 == 0: 75 | size = (t-12)/2 76 | if size > len(payload)-offset: 77 | size = len(payload)-offset 78 | v = unpack( str(size) + "s", payload[offset:offset+size] )[0] 79 | s = 'BLOB(' + str(size) + ')' 80 | offset += size 81 | elif t > 11 and t%2 == 1: 82 | size = (t-13)/2 83 | if size > len(payload)-offset: 84 | size = len(payload)-offset 85 | v = unpack( str(size) + "s", payload[offset:offset+size] )[0] 86 | s = 'STRING(' + str(size) + ')' 87 | offset += size 88 | f['strType'] = s 89 | f['value'] = v 90 | return fields 91 | 92 | def fileHeader(var, headerBytes): 93 | (var.String, var.pageSize, var.FFWv, var.FFRv, var.reservedSpace, var.maxPayload, 94 | var.minPayload, var.leafPayload, var.FileChangeCounter, var.DBSize, var.freelistTrunk, 95 | var.freelistTotal, var.SchemaCookie, var.SchemaFormat, var.PageCacheSize, var.VacuumRootPage, 96 | var.Encoding, var.UserVersion, var.VacuumMode, var.Reserved, var.VersionValid, var.Version) = unpack( ">16sH6B11I24s2I", headerBytes ) 97 | if var.Encoding == 1: 98 | var.EncodingStr = 'UTF-8' 99 | elif var.Encoding == 2: 100 | var.EncodingStr = 'UTF-16le' 101 | elif var.Encoding == 3: 102 | var.EncodingStr = 'UTF-16be' 103 | else: 104 | var.Encoding = 'Unknown' 105 | if var.pageSize == 1: 106 | var.pageSize = 65536 107 | var.usableSize = var.pageSize - var.reservedSpace 108 | var.maxLocal = (var.usableSize-12)*64/255 - 23 109 | var.minLocal = (var.usableSize-12)*32/255 - 23 110 | var.maxLeaf = var.usableSize - 35 111 | var.minLeaf = (var.usableSize-12)*32/255 - 23 112 | 113 | def isBTreePage( f ): 114 | if f == 0x05 or f == 0x0D or f == 0x02 or f == 0x0A: 115 | return 1 116 | return 0 117 | 118 | class SQLitePage(object): 119 | """Class to store complete Pages""" 120 | def __init__(self, pgn, parent, pageBytes): 121 | self.pageNum = pgn 122 | self.type = "Unknown" 123 | self.stream = pageBytes 124 | self.Flag = 0 125 | self.BTree = parent 126 | self.unallocated = ListRange( [0, parent.usableSize] ) 127 | self.cellGap = ListRange() 128 | self.isSubType = None 129 | 130 | def readCellHeader(self, cn): 131 | if cn < self.numCells: 132 | c = self.readCellHeaderOff( self.cells[cn]['offset'], cn ) 133 | 134 | def readCellHeaderOff(self, offset, cn=-1): 135 | cell = { 136 | 'pg': self.pageNum, 137 | 'offset': offset, 138 | 'leftChild': 0, 139 | 'payloadLen_v': 0, 140 | 'payloadLen': 0, 141 | 'key_v': 0, 142 | 'key': 0, 143 | 'payloadOffset': 0, 144 | 'localPayloadLen': 0, 145 | 'cellNo': None, 146 | 'cellSize': 0, 147 | 'overflowPage': None, 148 | 'overflowList': None, 149 | 'correct': True 150 | } 151 | cell['cellNo'] = cn 152 | of = 0 153 | if offset+of >= self.BTree.usableSize or offset+of+4 > self.BTree.usableSize: 154 | cell['correct'] = False 155 | return cell 156 | if not self.Flag & 0x08: 157 | cell['leftChild'] = unpack(">I", self.stream[offset+of:offset+of+4]) 158 | of += 4 159 | if self.Flag & 0x8 or self.Flag & 0x7 == 2: 160 | (cell['payloadLen'], cell['payloadLen_v']) = readVarInt( self.stream, offset+of ) 161 | of += cell['payloadLen_v'] 162 | if self.Flag & 0x1: 163 | (cell['key'], cell['key_v']) = readVarInt( self.stream, offset+of) 164 | of += cell['key_v'] 165 | if self.Flag & 0x8 or self.Flag & 0x7 == 2: 166 | cell['payloadOffset'] = of 167 | if cell['payloadLen'] > self.maxLocal: 168 | surplus = self.minLocal + (cell['payloadLen'] - self.minLocal)%(self.BTree.usableSize - 4) 169 | if surplus <= self.maxLocal: 170 | cell['localPayloadLen'] = surplus 171 | else: 172 | cell['localPayloadLen'] = self.minLocal 173 | cell['overflowPage'] = int(unpack(">I", self.stream[offset+of+cell['localPayloadLen']:offset+of+cell['localPayloadLen']+4])[0]) 174 | cell['overflowList'] = self.BTree.followOverflow( cell ) 175 | of += 4 176 | else: 177 | cell['localPayloadLen'] = cell['payloadLen'] 178 | cell['overflowPage'] = 0 179 | of += cell['localPayloadLen'] 180 | cell['cellSize'] = of 181 | return cell 182 | 183 | def dump(self): 184 | print"[+]Page", self.pageNum 185 | print"\tType:", self.type 186 | self.__dump__() 187 | 188 | def __dump__(self): 189 | pass 190 | 191 | 192 | class SQLiteBTreePage(SQLitePage): 193 | def __init__(self, pgn, parent, pageBytes, root=0): 194 | super(SQLiteBTreePage, self).__init__(pgn, parent, pageBytes) 195 | of = 0 196 | self.isFirstPage = root 197 | if self.isFirstPage: 198 | of += 100 199 | self.Flag = unpack(">B", self.stream[of:of+1])[0] 200 | of += 1 201 | if not isBTreePage (self.Flag): 202 | return 203 | self.FirstBlock = 0 204 | self.numnumCells = 0 205 | self.ContentArea = 0 206 | self.Fragment = 0 207 | self.avgCellSize = 0 208 | self.avgPayloadLen = 0 209 | self.unallocated = ListRange( ) 210 | self.Pointer = 0 211 | (self.FirstBlock, self.numCells, self.ContentArea, self.Fragment) = unpack(">3HB", self.stream[of:of+7]) 212 | self.maxLocal = self.BTree.maxLocal 213 | self.minLocal = self.BTree.minLocal 214 | of += 7 215 | if self.Flag & 0x8: 216 | self.strType = "Leaf" 217 | else: 218 | self.strType = "Interior" 219 | self.Pointer = unpack(">I", pageBytes[of:of+4])[0] 220 | of += 4 221 | if self.Flag & 0x7 == 2: 222 | self.strType += " Index"; 223 | elif self.Flag & 0x7 == 5: 224 | self.strType += " Table"; 225 | self.maxLocal = self.BTree.maxLeaf 226 | self.minLocal = self.BTree.minLeaf 227 | if self.strType != "Unknown": 228 | self.type = "BTree" 229 | if self.numCells: 230 | self.cells = [] 231 | self.cellGap.addRange( [self.ContentArea, self.BTree.usableSize] ) 232 | avgSize = 0 233 | avgPayload = 0 234 | for i in range(0, self.numCells): 235 | co = unpack( ">H", self.stream[of:of+2] )[0] 236 | cell = self.readCellHeaderOff(co, i) 237 | self.cells.append( cell ) 238 | self.cellGap.delRange( [cell['offset'], cell['offset']+cell['cellSize']] ) 239 | avgSize += cell['cellSize'] 240 | if 'payloadLen' in cell: 241 | avgPayload += cell['payloadLen'] 242 | of += 2 243 | for r in self.cellGap: 244 | self.unallocated.addRange( r ) 245 | self.avgCellSize = int(avgSize / self.numCells) 246 | self.avgPayloadLen = int(avgPayload / self.numCells) 247 | if self.ContentArea == self.BTree.usableSize: 248 | self.unallocated.addRange( [of, self.BTree.usableSize] ) 249 | else: 250 | self.unallocated.addRange( [of, self.ContentArea-1] ) 251 | 252 | def __dump__(self): 253 | if self.isFirstPage: 254 | print "\tFirst Page (with file header)" 255 | print "\tBTree Page Type", self.Flag, "(" + self.strType + ")" 256 | print "\tByte offset into First Freeblock:", self.FirstBlock 257 | print "\tNumber of cells:", self.numCells 258 | print "\tAvg. cell size:", self.avgCellSize 259 | if self.avgPayloadLen: 260 | print "\tAvg. payload size:", self.avgPayloadLen 261 | print "\tOffset to the first byte of content area:", self.ContentArea 262 | print "\tFragment:", self.Fragment 263 | if self.numCells: 264 | print "\tCell ptrs:", ",".join([ str(i['offset']) for i in self.cells]) 265 | if not(self.Flag & 0x8): 266 | print "\tThe right-most pointer:", self.Pointer 267 | print "\tUnallocated space:", ",".join([str(i) for i in self.unallocated]) 268 | print "\tCell gaps:", ",".join([str(i) for i in self.cellGap]) 269 | 270 | class SQLiteFreelistPage(SQLitePage): 271 | def __init__(self, pgn, parent, pageBytes, trunk=0): 272 | super(SQLiteFreelistPage, self).__init__(pgn, parent, pageBytes) 273 | self.isSubType = None 274 | self.isTrunk = trunk 275 | self.unallocated = ListRange( ) 276 | if trunk: 277 | self.type = 'Freelist Trunk' 278 | (self.nextTrunkPage, self.leafpageCount) = unpack( '>II', self.stream[0:8] ) 279 | self.leafPages = [] 280 | offset = 8 281 | for i in range(0, self.leafpageCount): 282 | self.leafPages.append( unpack( '>I', self.stream[offset:offset+4] )[0] ) 283 | offset += 4 284 | self.unallocated.addRange( [offset, parent.pageSize] ); 285 | else: 286 | #if isBTreePage( unpack(">B", pageBytes[0:1] )[0]): 287 | self.isSubType = SQLiteBTreePage(pgn, self.BTree, pageBytes) 288 | self.type = 'Freelist Leaf' 289 | self.unallocated.addRange( [1, parent.pageSize] ); 290 | 291 | def __dump__(self): 292 | if self.isTrunk: 293 | print "\tNext freelist trunk page:", self.nextTrunkPage 294 | print "\tLeaf pages count:", self.leafpageCount 295 | print "\tLeaf pages: " + ",".join(["%d" % i for i in self.leafPages ]) 296 | if self.isSubType: 297 | print "\tSubtype:", self.isSubType.type 298 | self.isSubType.__dump__( ) 299 | 300 | class SQLiteOverflowPage(SQLitePage): 301 | def __init__(self, pgn, parent, pageBytes, page=-1, cell=-1): 302 | super(SQLiteOverflowPage, self).__init__(pgn, parent, pageBytes) 303 | self.type = 'Cell Overflow' 304 | self.parentPage = page 305 | self.parentCell = cell 306 | self.nextOverflow = int(unpack(">I", self.stream[0:4])[0]) 307 | self.unallocated = ListRange( ) 308 | 309 | def __dump__(self): 310 | print "\tParent cell:", str(self.parentPage) + "," + str(self.parentCell) 311 | print "\tContinues on:", self.nextOverflow 312 | print "\tUnallocated space:", ",".join([str(i) for i in self.unallocated]) 313 | 314 | class SQLiteRaw: 315 | def __init__(self, stream): 316 | fileHeader(self, stream[0:100] ) 317 | self.stream = stream 318 | page1 = stream[:self.pageSize] 319 | pages = stream[self.pageSize:] 320 | self.Pages = { 1: SQLiteBTreePage(1, self, page1, 1) } 321 | if self.freelistTrunk: 322 | tp = self.freelistTrunk 323 | while tp: 324 | pgOffset = (tp-1)*self.pageSize 325 | freelistTP = SQLiteFreelistPage( tp, self, stream[pgOffset:pgOffset+self.pageSize], 1 ) 326 | self.Pages[tp] = freelistTP 327 | for pl in freelistTP.leafPages: 328 | plof = (pl-1)*self.pageSize 329 | self.Pages[pl] = SQLiteFreelistPage( pl, self, stream[plof:plof+self.pageSize] ) 330 | tp = freelistTP.nextTrunkPage 331 | 332 | self.pageCount = 1 333 | while len(pages) >= self.pageSize: 334 | self.pageCount += 1 335 | if not self.pageCount in self.Pages: 336 | try: 337 | self.Pages[self.pageCount] = SQLiteBTreePage( self.pageCount, self, pages[:self.pageSize] ) 338 | except: 339 | self.Pages[self.pageCount] = SQLitePage( self.pageCount, self, pages[:self.pageSize] ) 340 | pages = pages[self.pageSize:] 341 | def followOverflow(self, cell): 342 | if not cell['overflowPage']: 343 | return [] 344 | ret = [] 345 | overflow = cell['overflowPage'] 346 | size = cell['localPayloadLen'] 347 | while overflow: 348 | ret.append( overflow ) 349 | if not overflow in self.Pages: 350 | ovpg = SQLiteOverflowPage( overflow, self, self.stream[(overflow-1)*self.pageSize:overflow*self.pageSize], cell['pg'], cell['cellNo'] ) 351 | self.Pages[overflow] = ovpg 352 | else: 353 | ovpg = self.Pages[overflow] 354 | if ovpg.type == 'Overflow': 355 | overflow = ovpg.nextOverflow 356 | if overflow: 357 | size += self.usableSize-4 358 | else: 359 | ovpg.unallocated.addRange( [cell['payloadLen']-size+4, self.usableSize] ) 360 | else: 361 | break 362 | 363 | 364 | return ret 365 | 366 | def getCellPayload(self, cell): 367 | of = cell['offset'] + cell['payloadOffset'] 368 | read_len = l = cell['localPayloadLen'] 369 | stream = self.Pages[cell['pg']].stream 370 | payload = unpack( str(l) + "s", stream[of:of+l] )[0] 371 | if cell['overflowPage']: 372 | for ovpg in cell['overflowList']: 373 | stream = self.Pages[ovpg].stream 374 | l = min(self.usableSize-4, cell['payloadLen'] - read_len) 375 | payload += unpack( str(l) + "s", stream[4:4+l] )[0] 376 | return payload 377 | 378 | def dumpHeader(self): 379 | print "[+] Dumping SQLite Header" 380 | print "Header String:", self.String 381 | print "Page Size:",self.pageSize 382 | print "FFW, FFR:", self.FFWv, ",", self.FFRv 383 | print "Bytes of unused 'reserved' space:", self.reservedSpace 384 | print "Maximum embedded payload fraction (64):", self.maxPayload 385 | print "Minimun embedded payload fraction (32):", self.minPayload 386 | print "\tMaximum embedded payload:", self.maxLocal 387 | print "\tMinimun embedded payload:", self.minLocal 388 | print "Minimun Leaf payload (32):", self.leafPayload 389 | print "\tMaximum embedded Leaf payload:", self.maxLeaf 390 | print "\tMinumun embedded Leaf payload:", self.minLeaf 391 | print "File change counter:", self.FileChangeCounter 392 | print "Size of the database file in pages:", self.DBSize 393 | print "Page number of the first freelist trunk page:", self.freelistTrunk 394 | print "Total number of freelist pages:", self.freelistTotal 395 | print "The schema cookie:", self.SchemaCookie 396 | print "The schema format number (1, 2, 3 or 4):", self.SchemaFormat 397 | print "Default page cache size:", self.PageCacheSize 398 | print "Largest root b-tree page (Vacuum):", self.VacuumRootPage 399 | print "The database text encoding:", self.Encoding, "(" + self.EncodingStr +")" 400 | print "User Version:", self.UserVersion 401 | print "Incremental-Vacuum mode:", self.VacuumMode 402 | print "Reserved:", str(self.Reserved) 403 | print "Version valid for:", self.VersionValid 404 | print "SQLite Version:", self.Version 405 | 406 | def dump(self): 407 | self.dumpHeader() 408 | print "Real Page Count:", self.pageCount 409 | print "" 410 | def dump_pages(self): 411 | print "[+]Pages" 412 | for index, page in self.Pages.iteritems(): 413 | page.dump() 414 | 415 | -------------------------------------------------------------------------------- /recoversqlite.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | # Sat Jul 28 00:57:02 CEST 2012 4 | 5 | import sys 6 | import os 7 | import struct 8 | import getopt 9 | 10 | def usage ( ): 11 | print ('Usage: %s [options] -f \n' % sys.argv[0]) 12 | print ('Options:') 13 | print (' -h this help') 14 | print (' -v verbose') 15 | print (' -f sqlite file') 16 | print (' -o [h|a|n] output as hex (default), ascii, or nothing') 17 | 18 | sys.exit(1) 19 | 20 | 21 | ftype = "h" 22 | pages = "all" 23 | file = "" 24 | verbose = "0" 25 | 26 | try: 27 | opts, args = getopt.getopt(sys.argv[1:], "vho:f:") 28 | except getopt.GetoptError: 29 | usage() 30 | sys.exit(2) 31 | for opt, arg in opts: 32 | if opt in ('-h'): 33 | usage() 34 | elif opt == '-o': 35 | ftype = arg 36 | if ftype != "a" and ftype != "h" and ftype != "n": 37 | usage() 38 | elif opt == '-f': 39 | file = arg 40 | elif opt == '-v': 41 | verbose = 1 42 | 43 | if file == "": 44 | usage () 45 | 46 | if not os.path.exists(file): 47 | print 'error: file %s not found!' % file 48 | usage () 49 | 50 | 51 | if len(sys.argv) < 2: 52 | usage() 53 | sys.exit() 54 | 55 | filesize = os.path.getsize(file) 56 | with open(file, 'rb') as f: 57 | s = f.read() 58 | 59 | 60 | # The header string: "SQLite format 3\000" 61 | hs=s[:16].rstrip(' \t\r\n\0') 62 | if hs == "SQLite format 3": 63 | r = " (correct)" 64 | else: 65 | r = " (incorrect)" 66 | print '%-45s %-20s' % ("Header String:", hs + r) 67 | 68 | # The database page size in bytes. Must be a power of two between 512 69 | # and 32768 inclusive, or the value 1 representing a page size of 65536 70 | pagesize = struct.unpack(">H", s[16:18])[0] 71 | if pagesize == 1: 72 | pagesize == 65536 73 | print '%-45s %-20s' % ("Page Size bytes:", pagesize) 74 | 75 | # File format write version. 1 for legacy; 2 for WAL 76 | wver = ord(s[18:19]) 77 | if wver == 2: 78 | wrel = str(wver) + " - WAL: yes" 79 | else: 80 | wrel = str(wver) + " - WAL: no" 81 | print '%-45s %-20s' % ("File format write version:", wrel) 82 | 83 | # File format read version. 1 for legacy; 2 for WAL. 84 | rver = ord(s[19:20]) 85 | if rver == 2: 86 | rrel = str(rver) + " - WAL: yes" 87 | else: 88 | rrel = str(rver) + " - WAL: no" 89 | print '%-45s %-20s' % ("File format read version:", rrel) 90 | 91 | # Bytes of unused "reserved" space at the end of each page. Usually 0. 92 | if verbose == 1: print '%-45s %-20s' % ('Bytes of unused reserved space:', ord(s[20])) 93 | 94 | # Maximum embedded payload fraction. Must be 64. 95 | if ord(s[21]) == 64: 96 | r = " (correct)" 97 | else: 98 | r = " (incorrect)" 99 | if verbose == 1: print '%-45s %-20s' % ('Maximum embedded payload fraction:', str(ord(s[21])) + r) 100 | 101 | # Minimum embedded payload fraction. Must be 32. 102 | if ord(s[22]) == 32: 103 | r = " (correct)" 104 | else: 105 | r = " (incorrect)" 106 | if verbose == 1: print '%-45s %-20s' % ('Minimum embedded payload fraction:', str(ord(s[22])) + r) 107 | 108 | # Leaf payload fraction. Must be 32. 109 | if ord(s[23]) == 32: 110 | r = " (correct)" 111 | else: 112 | r = " (incorrect)" 113 | if verbose == 1: print '%-45s %-20s' % ('Leaf payload fraction:', str(ord(s[23])) + r) 114 | 115 | # File change counter. 116 | count = struct.unpack(">i", s[24:28])[0] 117 | print '%-45s %-20s' % ('File change counter:', count) 118 | 119 | # Size of the database file in pages. The "in-header database size". 120 | sizepag = struct.unpack(">i", s[28:32])[0] 121 | print '%-45s %-20s' % ('Size of the database file in pages:', sizepag) 122 | 123 | # Page number of the first freelist trunk page. 124 | pagenum = struct.unpack(">i", s[32:36])[0] 125 | print '%-45s %-20s' % ('Page number of the first freelist trunk page:', pagenum) 126 | 127 | # Total number of freelist pages. 128 | freenum = struct.unpack(">i", s[36:40])[0] 129 | print '%-45s %-20s' % ('Total number of freelist pages:', freenum) 130 | 131 | # The schema cookie. 132 | schema = struct.unpack(">i", s[40:44])[0] 133 | if verbose == 1: print '%-45s %-20s' % ('The schema cookie:', schema) 134 | 135 | # The schema format number. Supported schema formats are 1, 2, 3, and 4. 136 | schemav = struct.unpack(">i", s[44:48])[0] 137 | if schemav == 1: 138 | schemavs = " - back to ver 3.0.0" 139 | elif schemav == 2: 140 | schemavs = " - >= 3.1.3 on 2005-02-19" 141 | elif schemav == 3: 142 | schemavs = " - >= 3.1.4 on 2005-03-11" 143 | elif schemav == 4: 144 | schemavs = " - >= 3.3.0 on 2006-01-10" 145 | else: 146 | schemavs = " - schema version error" 147 | if verbose == 1: print '%-45s %-20s' % ('The schema format number', str(schemav) + schemavs) 148 | 149 | # Default page cache size. 150 | pcachesize = struct.unpack(">i", s[44:48])[0] 151 | if verbose == 1: print '%-45s %-20s' % ('Default page cache size:', pcachesize) 152 | 153 | # The page number of the largest root b-tree page when in auto-vacuum or incremental-vacuum modes, or zero otherwise. 154 | vacuum = struct.unpack(">i", s[52:56])[0] 155 | if vacuum == 0: 156 | vacuums = " - not supported" 157 | else: 158 | vacuums = " " 159 | if verbose == 1: print '%-45s %-20s' % ('Auto/incremental-vacuum page number:', str(vacuum) + vacuums) 160 | 161 | # The database text encoding. A value of 1 means UTF-8. A value of 2 means UTF-16le. A value of 3 means UTF-16be. 162 | encod = struct.unpack(">i", s[56:60])[0] 163 | if encod == 1: 164 | encods = " - UTF-8" 165 | elif encod == 2: 166 | encods = " - UTF-16le" 167 | elif encod == 3: 168 | encods = " - UTF-16be" 169 | else: 170 | encods = " - error" 171 | print '%-45s %-20s' % ('The database text encoding:', str(encod) + encods) 172 | 173 | # The "user version" as read and set by the user_version pragma. 174 | userv = struct.unpack(">i", s[60:64])[0] 175 | if verbose == 1: print '%-45s %-20s' % ('User version number:', userv) 176 | 177 | # True (non-zero) for incremental-vacuum mode. False (zero) otherwise. 178 | incvacuum = struct.unpack(">i", s[64:68])[0] 179 | if incvacuum == 0: 180 | sinc = " - false" 181 | else: 182 | sinc = " - true" 183 | print '%-45s %-20s' % ('Incremental-vacuum mode:', str(incvacuum) + sinc) 184 | 185 | reserv = struct.unpack(">iiiiii", s[68:92])[0] 186 | if reserv == 0: 187 | strreserv = "0 (correct)" 188 | else: 189 | strreserv = "!= 0 (incorrect)" 190 | if verbose == 1: print '%-45s %-20s' % ('Reserved for expansion:', strreserv) 191 | 192 | # The version-valid-for number. 193 | # The 4-byte big-endian integer at offset 92 is the value of the change counter 194 | # when the version number was stored. The integer at offset 92 indicates which 195 | # transaction the version number is valid for and is sometimes called the 196 | # "version-valid-for number". 197 | verval = struct.unpack(">i", s[92:96])[0] 198 | if verbose == 1: print '%-45s %-20s' % ('The version-valid-for number:', verval) 199 | 200 | # SQLITE_VERSION_NUMBER: 201 | # #define SQLITE_VERSION "3.7.13" 202 | # #define SQLITE_VERSION_NUMBER 3007013 203 | vervalid = struct.unpack(">i", s[96:100])[0] 204 | may = vervalid / 1000000 205 | min = (vervalid - (may * 1000000)) / 1000 206 | rls = vervalid - (may * 1000000) - (min * 1000) 207 | verstr = str(vervalid) + " - " + str(may) + "." + str(min) + "." + str(rls) 208 | print '%-45s %-20s' % ('SQLITE_VERSION_NUMBER:', verstr) 209 | 210 | def hexdump(src, length=16): 211 | if all_same(src): 212 | strzero = "0000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................\n" 213 | strzero += "..." 214 | return strzero 215 | result = [] 216 | digits = 4 if isinstance(src, unicode) else 2 217 | for i in xrange(0, len(src), length): 218 | s = src[i:i+length] 219 | hexa = b' '.join(["%0*X" % (digits, ord(x)) for x in s]) 220 | text = b'' 221 | for x in s: 222 | if(ord(x) < 0x20 or ord(x) == 0x7F or 0x81 <= ord(x) < 0xA0): 223 | text += b'.' 224 | else: 225 | text += x 226 | result.append( b"%04X %-*s |%s|" % (i, length*(digits + 1), 227 | hexa, text) ) 228 | return b'\n'.join(result) 229 | 230 | 231 | def asciidump(src, length=80): 232 | if all_same(src): 233 | strzero = "................................................................................" 234 | return strzero 235 | result = [] 236 | digits = 4 if isinstance(src, unicode) else 2 237 | for i in xrange(0, len(src), length): 238 | s = src[i:i+length] 239 | text = b'' 240 | for x in s: 241 | if(ord(x) < 0x20 or ord(x) == 0x7F or 0x81 <= ord(x) < 0xA0): 242 | text += b'.' 243 | else: 244 | text += x 245 | result.append( b"%s" % (text) ) 246 | return b'\n'.join(result) 247 | 248 | def all_same(items): 249 | return all(x == items[0] for x in items) 250 | 251 | def formatlist(list): 252 | i=0 253 | for el in list: 254 | i+=1 255 | if i == 10: 256 | if verbose == 1: print 257 | i=0 258 | else: 259 | if verbose == 1: print '%5d' % el, 260 | if verbose == 1: print 261 | 262 | 263 | 264 | def locatetrunk ( offset ): 265 | nextrunk = struct.unpack(">i", s[offset:offset+4])[0] 266 | if nextrunk != 0: 267 | return nextrunk 268 | else: 269 | return 0 270 | 271 | def locatefreeleafs ( offset ): 272 | numleafpag = struct.unpack(">i", s[offset+4:offset+8])[0] 273 | # -24 -> bug in sqlite avoids writing in last 6 entries 274 | freepage = s[offset+8:offset+pagesize-24] 275 | fmt = ">" + "i" * (len(freepage)/4) 276 | # return only numleafpag 277 | return struct.unpack(fmt, freepage)[0:numleafpag] 278 | 279 | 280 | def freepages( ): 281 | offset = (pagenum - 1)*pagesize 282 | freeleaftpages=locatefreeleafs( offset ) 283 | nextrunk = 1 284 | freetrunk.append(pagenum) 285 | while ( nextrunk != 0 ): 286 | nextrunk = locatetrunk( offset ) 287 | if nextrunk != 0: 288 | freetrunk.append(nextrunk) 289 | offset = (nextrunk - 1)*pagesize 290 | freeleaftpages += locatefreeleafs( offset ) 291 | 292 | freeleaf = list(set(freeleaftpages)) 293 | 294 | return freeleaf, freetrunk 295 | 296 | def locatebtree( ): 297 | offset = 100 298 | page = 1 299 | while ( offset < filesize ): 300 | byte = ord(s[offset]) 301 | if byte == 13 and page not in freeleaf: 302 | leafpages.append(page) 303 | elif byte == 2 and page not in freeleaf: 304 | interindex.append(page) 305 | elif byte == 5 and page not in freeleaf: 306 | intertable.append(page) 307 | elif byte == 10 and page not in freeleaf: 308 | leafindex.append(page) 309 | if offset == 100: 310 | offset = 0 311 | offset += pagesize 312 | page += 1 313 | 314 | def pagedump(pages): 315 | for page in pages: 316 | offset = (page-1 )*pagesize 317 | end = 0 318 | if page == 0 or offset == 0: 319 | offset += 100 320 | end = 100 321 | if page not in freeleaf and page not in freetrunk: 322 | # B-Tree header 323 | # 324 | # Number of cells on this page 325 | numcells = struct.unpack(">H", s[offset+3:offset+5])[0] 326 | # Offset to the first byte of the cell content area. 327 | # A zero value is used to represent an offset of 65536, 328 | # which occurs on an empty root page when using a 65536-byte page size. 329 | offcells = struct.unpack(">H", s[offset+5:offset+7])[0] 330 | osfree = struct.unpack(">H", s[offset+1:offset+3])[0] 331 | fragmented = struct.unpack(">b", s[offset+7])[0] 332 | nextfb = osfree 333 | fbsize = 0 334 | if offcells == 0: 335 | offcells = 65536 336 | # interior btree header = 12 bytes, leaf btree = 8 bytes 337 | head = 8 338 | if ord(s[offset]) == 2 or 5: head = 12 339 | # unallocated start and end 340 | unstart = offset + head + ( numcells * 2 ) 341 | unend = offset + offcells - end 342 | freestr = s[unstart:unend] 343 | if ftype != "n": 344 | print '%-25s page: %s offset: %10s-%-10s' % ('Unallocated space: ', page, unstart,unend) 345 | if ftype == "a": 346 | print asciidump(freestr) 347 | elif ftype == "h": 348 | print hexdump(freestr) 349 | 350 | while ( nextfb != 0 ): 351 | # freeblock 1,2 bytes next block or 0 if none 352 | # freeblock 3,4 size of freeblock 353 | fbsize = struct.unpack(">H", s[offset+nextfb+2:offset+nextfb+4])[0] 354 | fbstart = offset+nextfb+4 355 | fbend = offset+nextfb+fbsize 356 | fbstr = s[fbstart:fbend] 357 | if ftype != "n": 358 | print '%-25s page: %s offset: %10s-%-10s' % ('Freeblock space: ', page, fbstart,fbend) 359 | if ftype == "a": 360 | print asciidump(fbstr) 361 | elif ftype == "h": 362 | print hexdump(fbstr) 363 | nextfb = struct.unpack(">H", s[offset+nextfb:offset+nextfb+2])[0] 364 | elif page in freeleaf: 365 | freestr = s[offset:offset+pagesize] 366 | if ftype != "n": 367 | print '%-25s page: %s offset: %10s' % ('Freelist leaf page: ',page, offset) 368 | if ftype == "a": 369 | print asciidump(freestr) 370 | elif ftype == "h": 371 | print hexdump(freestr) 372 | 373 | elif page in freetrunk: 374 | ftstart = offset+8+(4*len(freeleaf)) 375 | ftend = offset+pagesize 376 | freestr = s[ftstart:ftend] 377 | if ftype != "n": 378 | print '%-25s page: %s offset: %10s-%-10s' % ('Freelist trunk space: ', page, ftstart,ftend) 379 | if ftype == "a": 380 | print asciidump(freestr) 381 | elif ftype == "h": 382 | print hexdump(freestr) 383 | 384 | 385 | 386 | 387 | 388 | # Freepages 389 | freeleaf = [ ] 390 | freetrunk = [ ] 391 | 392 | if freenum > 0: freeleaf, freetrunk = freepages( ) 393 | if verbose == 1 and freenum >0: 394 | print "Freelist leaf pages:" 395 | formatlist(freeleaf) 396 | print '%-45s %-20s' % ('Number of freelist leaf pages:', len(sorted(set(freeleaf)))) 397 | 398 | ## B-tree pages unallocated + freeblocks 399 | btreepages = [ ] 400 | leafpages = [ ] 401 | interindex = [ ] 402 | intertable = [ ] 403 | leafindex = [ ] 404 | 405 | 406 | locatebtree() 407 | 408 | 409 | if verbose == 1 and len(freetrunk) >0: 410 | print "Freelist trunk pages:" 411 | formatlist(freetrunk) 412 | print '%-45s %-20s' % ('Number of freelist trunk pages:', len(freetrunk)) 413 | 414 | if verbose == 1 and len(leafpages) > 0: 415 | print "B-Tree leaf pages:" 416 | formatlist(leafpages) 417 | print '%-45s %-20s' % ('Number of b-tree leaf pages:', len(leafpages)) 418 | if verbose == 1 and len(leafindex) >0: 419 | print "B-Tree leaf index pages:" 420 | formatlist(leafindex) 421 | print '%-45s %-20s' % ('Number of b-tree leaf index pages:', len(leafindex)) 422 | if verbose == 1 and len(interindex) >0: 423 | print "B-Tree interior index pages:" 424 | formatlist(interindex) 425 | print '%-45s %-20s' % ('Number of b-tree interior index pages:', len(interindex)) 426 | if verbose == 1 and len(intertable) >0: 427 | print "B-Tree interior table pages:" 428 | formatlist(intertable) 429 | print '%-45s %-20s' % ('Number of b-tree interior table pages:', len(intertable)) 430 | 431 | 432 | btreepages = sorted(leafpages + leafindex + interindex + intertable) 433 | allpag = sorted(btreepages + freeleaf + freetrunk) 434 | 435 | pagedump(allpag) 436 | 437 | 438 | if verbose == 1: 439 | print '%-45s %-20s' % ('Number of detected pages:', len(allpag)) 440 | print '%-45s %-20s' % ('Missing:', sizepag - len(allpag)) 441 | 442 | 443 | --------------------------------------------------------------------------------