├── .travis.yml
├── AUTHORS
├── CONTRIBUTORS
├── LICENSE
├── README.md
├── btree.go
├── btree_test.go
├── cell.go
├── cmd
    └── sqlite-dump
    │   └── main.go
├── file.go
├── page.go
├── pager.go
├── record.go
├── sqlite.go
├── sqlite_test.go
├── stypes.go
├── table.go
├── testdata
    ├── all-int-types.sqlite
    ├── chrome-history.sqlite
    ├── firefox-history.sqlite
    ├── safari-partial.sqlite
    ├── test-1.sqlite
    └── test-2.sqlite
├── utils.go
└── utils_test.go


/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: go
 2 | go:
 3 |   - 1.7.x
 4 |   - 1.8.x
 5 |   - 1.9.x
 6 |   - master
 7 | os:
 8 |   - linux
 9 | matrix:
10 |  fast_finish: true
11 |  allow_failures:
12 |    - go: master
13 | 
14 | sudo: false
15 | 
16 | notifications:
17 |   email:
18 |     recipients:
19 |       - seb.binet@gmail.com
20 |     on_success: always
21 |     on_failure: always
22 | 
23 | 


--------------------------------------------------------------------------------
/AUTHORS:
--------------------------------------------------------------------------------
 1 | # This is the official list of go-sqlite authors for copyright purposes.
 2 | # This file is distinct from the CONTRIBUTORS files.
 3 | # See the latter for an explanation.
 4 | 
 5 | # Names should be added to this file as
 6 | #	Name or Organization <email address>
 7 | # The email address is not required for organizations.
 8 | 
 9 | # Please keep the list sorted.
10 | 
11 | Adam Shannon <adamkshannon@gmail.com>
12 | Andre Renaud <andre@ignavus.net>
13 | Sebastien Binet <binet@cern.ch> <seb.binet@gmail.com>
14 | Zellyn Hunter <zellyn@gmail.com>
15 | 


--------------------------------------------------------------------------------
/CONTRIBUTORS:
--------------------------------------------------------------------------------
 1 | # This is the official list of people who can contribute
 2 | # (and typically have contributed) code to the go-sqlite
 3 | # repository.
 4 | #
 5 | # The AUTHORS file lists the copyright holders; this file
 6 | # lists people.  For example, ACME Inc. employees would be listed here
 7 | # but not in AUTHORS, because ACME Inc. would hold the copyright.
 8 | #
 9 | # When adding J Random Contributor's name to this file,
10 | # either J's name or J's organization's name should be
11 | # added to the AUTHORS file.
12 | #
13 | # Names should be added to this file like so:
14 | #     Name <email address>
15 | #
16 | # Please keep the list sorted.
17 | 
18 | Adam Shannon <adamkshannon@gmail.com>
19 | Andre Renaud <andre@ignavus.net>
20 | Sebastien Binet <binet@cern.ch> <seb.binet@gmail.com>
21 | Zellyn Hunter <zellyn@gmail.com>
22 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright ©2013 The go-sqlite Authors. All rights reserved.
 2 | 
 3 | Redistribution and use in source and binary forms, with or without
 4 | modification, are permitted provided that the following conditions are met:
 5 |     * Redistributions of source code must retain the above copyright
 6 |       notice, this list of conditions and the following disclaimer.
 7 |     * Redistributions in binary form must reproduce the above copyright
 8 |       notice, this list of conditions and the following disclaimer in the
 9 |       documentation and/or other materials provided with the distribution.
10 |     * Neither the name of the go-sqlite project nor the names of its authors and
11 |       contributors may be used to endorse or promote products derived from this
12 |       software without specific prior written permission.
13 | 
14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # sqlite3
 2 | 
 3 | [![GoDoc](https://godoc.org/github.com/go-sqlite/sqlite3?status.svg)](https://godoc.org/github.com/go-sqlite/sqlite3)
 4 | [![Build Status](https://travis-ci.org/go-sqlite/sqlite3.svg?branch=master)](https://travis-ci.org/go-sqlite/sqlite3)
 5 | 
 6 | `sqlite3` is a pure Go package decoding the `SQLite` file format as
 7 | described by:
 8 |  http://www.sqlite.org/fileformat.html
 9 | 
10 | ## Current status
11 | 
12 | **WIP**: The near-term aim for `sqlite3` is to iterate through the
13 | data in tables in `SQLite` files: ie., readonly access, and no actual
14 | SQL queries.
15 | 
16 | It doesn't quite do that yet: so far it just parses the
17 | `sqlite_master` data enough to find a list of tables and their names.
18 | 
19 | ## Installation
20 | 
21 | ```sh
22 | $ go get github.com/go-sqlite/sqlite3
23 | ```
24 | 
25 | ## License
26 | 
27 | `sqlite3` is released under the `BSD-3` license.
28 | 
29 | 
30 | ## Example
31 | 
32 | ```go
33 | package main
34 | 
35 | import (
36 | 	"fmt"
37 | 
38 | 	"github.com/go-sqlite/sqlite3"
39 | )
40 | 
41 | func main() {
42 | 	db, err := sqlite3.Open("test.sqlite")
43 | 	if err != nil {
44 | 		panic(err)
45 | 	}
46 | 	defer db.Close()
47 | 
48 | 	for _, table := range db.Tables() {
49 | 		fmt.Printf(">>> table=%#v\n", table)
50 | 	}
51 | }
52 | ```
53 | 
54 | ## Contributing
55 | 
56 | We're always looking for new contributing finding bugs, fixing issues, or writing some docs. If you're interested in contriburing source code changes you'll just need to [pull down the source code](#installation). You can run tests with `go test ./...` in the root of this project.
57 | 
58 | Make sure to add yourself to `AUTHORS` and `CONTRIBUTORS` if you submit a PR. We want you to take credit for your work!
59 | 


--------------------------------------------------------------------------------
/btree.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2017 The go-sqlite Authors. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | package sqlite3
  6 | 
  7 | import (
  8 | 	"fmt"
  9 | 	"io"
 10 | 	"strings"
 11 | 
 12 | 	"github.com/gonuts/binary"
 13 | )
 14 | 
 15 | type btheader struct {
 16 | 	raw struct {
 17 | 		Kind PageKind // b-tree page kind
 18 | 
 19 | 		FreeBlockOffset int16 // byte offset into the page of the first free block
 20 | 		NCells          int16 // number of cells on this page
 21 | 		CellsOffset     int16 // offset into first byte of the cell content area
 22 | 		NFreeBytes      int8  // number of fragmented free bytes within cell area content
 23 | 	}
 24 | }
 25 | 
 26 | func (bt btheader) Kind() PageKind {
 27 | 	return bt.raw.Kind
 28 | }
 29 | 
 30 | func (bt *btheader) NumCell() int {
 31 | 	return int(bt.raw.NCells)
 32 | }
 33 | 
 34 | func (bt *btheader) FreeBlockAddr() int {
 35 | 	return int(bt.raw.FreeBlockOffset)
 36 | }
 37 | 
 38 | func (bt *btheader) CellsAddr() int {
 39 | 	return int(bt.raw.CellsOffset)
 40 | }
 41 | 
 42 | type btreeTable struct {
 43 | 	btheader
 44 | 	db      *DbFile
 45 | 	pointer int32   // right most pointer (only valid for interior pages
 46 | 	page    page    // page backing this b-tree leaf
 47 | 	addrs   []int16 // cell addresses
 48 | }
 49 | 
 50 | func newBtreeTable(page page, db *DbFile) (*btreeTable, error) {
 51 | 	var hdr btheader
 52 | 	if page.ID() == 1 {
 53 | 		// drop first 100-bytes (global file header)
 54 | 		_, err := page.Seek(100, 0)
 55 | 		if err != nil {
 56 | 			return nil, err
 57 | 		}
 58 | 	}
 59 | 
 60 | 	err := page.Decode(&hdr.raw)
 61 | 	if err != nil {
 62 | 		return nil, err
 63 | 	}
 64 | 
 65 | 	btree := &btreeTable{
 66 | 		btheader: hdr,
 67 | 		db:       db,
 68 | 		page:     page,
 69 | 	}
 70 | 
 71 | 	if btree.Kind() == BTreeInteriorTableKind {
 72 | 		err = btree.page.Decode(&btree.pointer)
 73 | 		if err != nil {
 74 | 			return nil, err
 75 | 		}
 76 | 	}
 77 | 
 78 | 	err = btree.init()
 79 | 	if err != nil {
 80 | 		return nil, err
 81 | 	}
 82 | 
 83 | 	return btree, err
 84 | }
 85 | 
 86 | func (btree *btreeTable) ID() int {
 87 | 	return btree.page.ID()
 88 | }
 89 | 
 90 | func (btree *btreeTable) Size() int {
 91 | 	return len(btree.page.buf)
 92 | }
 93 | 
 94 | func (btree *btreeTable) init() error {
 95 | 	var err error
 96 | 	if btree.addrs != nil {
 97 | 		return nil
 98 | 	}
 99 | 
100 | 	cells := make([]int16, btree.NumCell())
101 | 	for icell, addr := range cells {
102 | 		// fmt.Printf("   cell= %d/%d... (%d)\n", icell+1, len(cells), btree.page.Pos())
103 | 		err = btree.page.Decode(&addr)
104 | 		if err != nil {
105 | 			return err
106 | 		}
107 | 		// fmt.Printf("   cell= %d/%d... (%d) => %d\n", icell+1, len(cells), btree.page.Pos(), addr)
108 | 		cells[icell] = addr
109 | 	}
110 | 
111 | 	btree.addrs = cells
112 | 	return err
113 | }
114 | 
115 | func (btree *btreeTable) decodeRecord(payload []byte) (Record, error) {
116 | 	var rec Record
117 | 
118 | 	// decode record
119 | 	recbuf := payload[:]
120 | 	rhdrsz, n := varint(recbuf)
121 | 	if n <= 0 {
122 | 		return rec, fmt.Errorf("sqlite3: error decoding record header (n=%d)", n)
123 | 	}
124 | 	recbuf = recbuf[n:]
125 | 
126 | 	rec.Header.Len = int(rhdrsz) - n
127 | 	for ii := 0; ii < rec.Header.Len; {
128 | 		v, n := varint(recbuf)
129 | 		// fmt.Printf("ii=%d nn=%d len=%d\n", ii, n, rec.Header.Len)
130 | 		if n <= 0 {
131 | 			return rec, fmt.Errorf("sqlite3: error decoding record header type (n=%d)", n)
132 | 		}
133 | 		recbuf = recbuf[n:]
134 | 		ii += int(n)
135 | 		rec.Header.Types = append(rec.Header.Types, SerialType(v))
136 | 	}
137 | 	rec.Body = recbuf[:]
138 | 	//copy(rec.Body, recbuf)
139 | 
140 | 	// fmt.Printf(">>> record: %#v (body=%d)\n", rec.Header, len(rec.Body))
141 | 	for _, st := range rec.Header.Types {
142 | 		var v interface{}
143 | 		switch st {
144 | 		case StInt8:
145 | 			recbuf, v = readStInt8(recbuf)
146 | 
147 | 		case StInt16:
148 | 			recbuf, v = readStInt16(recbuf)
149 | 
150 | 		case StInt24:
151 | 			recbuf, v = readStInt24(recbuf)
152 | 
153 | 		case StInt32:
154 | 			recbuf, v = readStInt32(recbuf)
155 | 
156 | 		case StInt48:
157 | 			recbuf, v = readStInt48(recbuf)
158 | 
159 | 		case StInt64:
160 | 			recbuf, v = readStInt64(recbuf)
161 | 
162 | 		case StFloat:
163 | 			var vv float64
164 | 			n, err := unmarshal(recbuf, &vv)
165 | 			if err != nil {
166 | 				panic(err)
167 | 			}
168 | 			recbuf = recbuf[int(n):]
169 | 			v = vv
170 | 
171 | 		case StC0:
172 | 			v = 0
173 | 
174 | 		case StC1:
175 | 			v = 1
176 | 
177 | 		default:
178 | 			if st.IsBlob() {
179 | 				vv := make([]byte, st.NBytes())
180 | 				n := copy(vv, recbuf)
181 | 				recbuf = recbuf[int(n):]
182 | 				v = vv
183 | 			}
184 | 			if st.IsText() {
185 | 				vv := make([]byte, st.NBytes())
186 | 				n := copy(vv, recbuf)
187 | 				recbuf = recbuf[int(n):]
188 | 				// FIXME(sbinet)
189 | 				// handle db string encoding
190 | 				switch btree.db.header.DbEncoding {
191 | 				case 1:
192 | 					s := string(vv)
193 | 					idx := strings.Index(s, "\x00")
194 | 					if idx >= 0 {
195 | 						s = s[:idx]
196 | 					}
197 | 					v = s
198 | 				default:
199 | 					panic("utf-16 not supported")
200 | 				}
201 | 			}
202 | 		}
203 | 
204 | 		rec.Values = append(rec.Values, v)
205 | 	}
206 | 	// fmt.Printf(">>> record: %#v (body=%d)\n", rec.Values, len(rec.Body))
207 | 
208 | 	return rec, nil
209 | }
210 | 
211 | func (btree *btreeTable) loadCell(icell int) (cellInfo, error) {
212 | 	addr := btree.addrs[icell]
213 | 	if _, err := btree.page.Seek(int64(addr), 0); err != nil {
214 | 		return cellInfo{}, err
215 | 	}
216 | 	return btree.parseCell()
217 | }
218 | 
219 | func (btree *btreeTable) parseCell() (cellInfo, error) {
220 | 	var cell cellInfo
221 | 	var err error
222 | 
223 | 	switch btree.Kind() {
224 | 	case BTreeInteriorIndexKind:
225 | 		panic("not implemented")
226 | 	case BTreeInteriorTableKind:
227 | 		var pgno int32 // page number of left child
228 | 		err = btree.page.Decode(&pgno)
229 | 		if err != nil {
230 | 			return cell, fmt.Errorf("sqlite3: error decoding page number: %v", err)
231 | 		}
232 | 
233 | 		rowid, nrow := btree.page.Varint()
234 | 		if nrow <= 0 {
235 | 			return cell, fmt.Errorf("sqlite3: error decoding rowid: n=%d", nrow)
236 | 		}
237 | 
238 | 		signedRowid := int64(rowid)
239 | 
240 | 		cell = cellInfo{
241 | 			LeftChildPage: int32(pgno),
242 | 			RowID:         &signedRowid,
243 | 		}
244 | 		// fmt.Printf(">>> cell: %#v\n", cell)
245 | 
246 | 	case BTreeLeafIndexKind:
247 | 		panic("not implemented")
248 | 	case BTreeLeafTableKind:
249 | 		sz, nsz := btree.page.Varint()
250 | 		if nsz <= 0 {
251 | 			return cell, fmt.Errorf("sqlite3: error decoding cell size: n=%d", nsz)
252 | 		}
253 | 
254 | 		rowid, nrow := btree.page.Varint()
255 | 		if nrow <= 0 {
256 | 			return cell, fmt.Errorf("sqlite3: error decoding rowid: n=%d", nrow)
257 | 		}
258 | 
259 | 		signedRowid := int64(rowid)
260 | 
261 | 		// sz is the total payload size.
262 | 		// check if all of it is in the b-tree leaf page or
263 | 		// if it spilled over to other pages
264 | 		P := int(sz)
265 | 		U := btree.page.PageSize() - int(btree.db.header.NReserved)
266 | 		X := U - 35
267 | 		localsz := P
268 | 		overflowsz := 0
269 | 		if P > X {
270 | 			M := int(((U - 12) * 32 / 255) - 23)
271 | 			K := M + ((P - M) % (U - 4))
272 | 			localsz = K
273 | 			if K > X {
274 | 				localsz = M
275 | 			}
276 | 			overflowsz = P - localsz
277 | 		}
278 | 
279 | 		// FIXME(sbinet): only create a new payload []byte when non-local
280 | 		// ie: when there is an overflow page
281 | 		payload := make([]byte, localsz, localsz)
282 | 		_, err := io.ReadFull(&btree.page, payload)
283 | 		if err != nil {
284 | 			return cell, err
285 | 		}
286 | 
287 | 		cell = cellInfo{
288 | 			RowID:   &signedRowid,
289 | 			Payload: payload,
290 | 		}
291 | 
292 | 		if localsz != P {
293 | 			err = btree.page.Decode(&cell.OverflowPage)
294 | 			if err != nil {
295 | 				return cell, err
296 | 			}
297 | 
298 | 			overflow, err := btree.readOverflow(cell.OverflowPage, overflowsz)
299 | 			if err != nil {
300 | 				return cell, err
301 | 			}
302 | 			cell.Payload = append(cell.Payload, overflow...)
303 | 		}
304 | 
305 | 		if len(cell.Payload) != int(sz) {
306 | 			panic(fmt.Errorf("read %d payload bytes instead of %d", len(cell.Payload), sz))
307 | 		}
308 | 
309 | 		// fmt.Printf(" => size=%d rowid=%d overflow=%d (bytes: %d %d) [%d]\n",
310 | 		// 	cell.Len,
311 | 		// 	cell.RowID,
312 | 		// 	cell.OverflowPage,
313 | 		// 	nsz, nrow,
314 | 		// 	btree.page.Pos(),
315 | 		// )
316 | 		// fmt.Printf(" => %x (%d|%d)\n", string(cell.Payload), len(cell.Payload), localsz)
317 | 
318 | 	}
319 | 	return cell, err
320 | }
321 | 
322 | // readOverflow reads `size` overflow page bytes, starting at page
323 | // `pageNum`, following the linked list of overflow pages as
324 | // necessary.
325 | func (btree *btreeTable) readOverflow(pageNum int32, size int) ([]byte, error) {
326 | 	usable := btree.page.PageSize() - int(btree.db.header.NReserved)
327 | 	sizeLeft := size
328 | 
329 | 	result := make([]byte, 0, size)
330 | 
331 | 	for pageNum != 0 {
332 | 		if sizeLeft == 0 {
333 | 			return nil, fmt.Errorf("read all %d bytes but still have overflow page %d", size, pageNum)
334 | 		}
335 | 		page, err := btree.db.pager.Page(int(pageNum))
336 | 		if err != nil {
337 | 			return nil, err
338 | 		}
339 | 
340 | 		err = page.Decode(&pageNum)
341 | 		if err != nil {
342 | 			return nil, err
343 | 		}
344 | 
345 | 		buf := make([]byte, min(sizeLeft, usable-4))
346 | 		n, err := page.Read(buf)
347 | 		if err != nil {
348 | 			return nil, err
349 | 		}
350 | 
351 | 		result = append(result, buf...)
352 | 		sizeLeft = sizeLeft - n
353 | 	}
354 | 
355 | 	if sizeLeft != 0 {
356 | 		return nil, fmt.Errorf("ran out of overflow pages with %d of %d bytes left unread", sizeLeft, size)
357 | 	}
358 | 
359 | 	if len(result) != size {
360 | 		panic(fmt.Errorf("read %d overflow bytes instead of %d", len(result), size))
361 | 	}
362 | 	return result, nil
363 | }
364 | 
365 | // Perform inorder traversal of all cells in the btree and its
366 | // children, passing each raw cell to the visitor function `f`.
367 | func (btree *btreeTable) visitRawInorder(f func(cellInfo) error) error {
368 | 	btreeHasData := btree.Kind() != BTreeInteriorTableKind
369 | 
370 | 	for i := 0; i < btree.NumCell(); i++ {
371 | 		cell, err := btree.loadCell(i)
372 | 		if err != nil {
373 | 			return err
374 | 		}
375 | 
376 | 		if cell.LeftChildPage != 0 {
377 | 			page, err := btree.db.pager.Page(int(cell.LeftChildPage))
378 | 			if err != nil {
379 | 				return err
380 | 			}
381 | 
382 | 			childBtree, err := newBtreeTable(page, btree.db)
383 | 			if err != nil {
384 | 				return err
385 | 			}
386 | 
387 | 			if err := childBtree.visitRawInorder(f); err != nil {
388 | 				return err
389 | 			}
390 | 		}
391 | 
392 | 		// Skip the call to f for interior table btrees: they have no
393 | 		// actual data.
394 | 		if btreeHasData {
395 | 			if err := f(cell); err != nil {
396 | 				return err
397 | 			}
398 | 		}
399 | 	}
400 | 
401 | 	if btree.pointer != 0 {
402 | 		page, err := btree.db.pager.Page(int(btree.pointer))
403 | 		if err != nil {
404 | 			return err
405 | 		}
406 | 
407 | 		childBtree, err := newBtreeTable(page, btree.db)
408 | 		if err != nil {
409 | 			return err
410 | 		}
411 | 
412 | 		if err := childBtree.visitRawInorder(f); err != nil {
413 | 			return err
414 | 		}
415 | 	}
416 | 
417 | 	return nil
418 | }
419 | 
420 | // Perform inorder traversal of all cells in the btree and its
421 | // children, passing the record-decoded payload of each cell to the
422 | // visitor function `f`.
423 | func (btree *btreeTable) visitRecordsInorder(f func(*int64, Record) error) error {
424 | 	return btree.visitRawInorder(func(ci cellInfo) error {
425 | 		if len(ci.Payload) == 0 {
426 | 			return nil
427 | 		}
428 | 		if rec, err := btree.decodeRecord(ci.Payload); err != nil {
429 | 			return err
430 | 		} else {
431 | 			return f(ci.RowID, rec)
432 | 		}
433 | 	})
434 | }
435 | 
436 | func readStInt8(buf []byte) ([]byte, int8) {
437 | 	var v int8
438 | 	n, err := unmarshal(buf, &v)
439 | 	if err != nil {
440 | 		panic(err)
441 | 	}
442 | 	return buf[int(n):], v
443 | }
444 | 
445 | func readStInt16(buf []byte) ([]byte, int16) {
446 | 	var v int16
447 | 	n, err := unmarshal(buf, &v)
448 | 	if err != nil {
449 | 		panic(err)
450 | 	}
451 | 	return buf[int(n):], v
452 | }
453 | 
454 | func readStInt24(buf []byte) ([]byte, uint32) {
455 | 	bs := make([]byte, 4)
456 | 	if n := copy(bs[1:], buf); n != 3 {
457 | 		panic(fmt.Sprintf("read %d bytes", n))
458 | 	}
459 | 	if bs[1]&0x80 > 0 {
460 | 		bs[0] = 0xff
461 | 	}
462 | 	return buf[3:], binary.BigEndian.Uint32(bs)
463 | }
464 | 
465 | func readStInt32(buf []byte) ([]byte, int32) {
466 | 	var v int32
467 | 	n, err := unmarshal(buf, &v)
468 | 	if err != nil {
469 | 		panic(err)
470 | 	}
471 | 	return buf[int(n):], v
472 | }
473 | 
474 | func readStInt48(buf []byte) ([]byte, uint64) {
475 | 	bs := make([]byte, 8)
476 | 	if n := copy(bs[2:], buf); n != 6 {
477 | 		panic(fmt.Sprintf("read %d bytes", n))
478 | 	}
479 | 	if bs[2]&0x80 > 0 {
480 | 		bs[0] = 0xff
481 | 	}
482 | 	return buf[6:], binary.BigEndian.Uint64(bs)
483 | }
484 | 
485 | func readStInt64(buf []byte) ([]byte, int64) {
486 | 	var v int64
487 | 	n, err := unmarshal(buf, &v)
488 | 	if err != nil {
489 | 		panic(err)
490 | 	}
491 | 	return buf[int(n):], v
492 | }
493 | 


--------------------------------------------------------------------------------
/btree_test.go:
--------------------------------------------------------------------------------
  1 | package sqlite3
  2 | 
  3 | import (
  4 | 	"reflect"
  5 | 	"testing"
  6 | )
  7 | 
  8 | func TestBTree__readStInt8(t *testing.T) {
  9 | 	rbuf, res := readStInt8([]byte{0x7f})
 10 | 	if len(rbuf) != 0 {
 11 | 		t.Errorf("len(rbuf)=%d", len(rbuf))
 12 | 	}
 13 | 	ans := int8(1<<7-1)
 14 | 	if res != ans {
 15 | 		t.Errorf("got %d, expected %d", res, ans)
 16 | 	}
 17 | }
 18 | 
 19 | func TestBTree__readStInt16(t *testing.T) {
 20 | 	rbuf, res := readStInt16([]byte{0x7f, 0xff})
 21 | 	if len(rbuf) != 0 {
 22 | 		t.Errorf("len(rbuf)=%d", len(rbuf))
 23 | 	}
 24 | 	ans := int16(1<<15-1)
 25 | 	if res != ans {
 26 | 		t.Errorf("got %d, expected %d", res, ans)
 27 | 	}
 28 | }
 29 | 
 30 | func TestBTree__readStInt24(t *testing.T) {
 31 | 	rbuf, res := readStInt24([]byte{0x7f, 0xff, 0xff})
 32 | 	if len(rbuf) != 0 {
 33 | 		t.Errorf("len(rbuf)=%d", len(rbuf))
 34 | 	}
 35 | 	ans := uint32(1<<23-1)
 36 | 	if res != ans {
 37 | 		t.Errorf("got %d, expected %d", res, ans)
 38 | 	}
 39 | }
 40 | 
 41 | func TestBTree__readStInt32(t *testing.T) {
 42 | 	rbuf, res := readStInt32([]byte{0x7f, 0xff, 0xff, 0xff})
 43 | 	if len(rbuf) != 0 {
 44 | 		t.Errorf("len(rbuf)=%d", len(rbuf))
 45 | 	}
 46 | 	ans := int32(1<<31-1)
 47 | 	if res != ans {
 48 | 		t.Errorf("got %d, expected %d", res, ans)
 49 | 	}
 50 | }
 51 | 
 52 | func TestBTree__readStInt48(t *testing.T) {
 53 | 	rbuf, res := readStInt48([]byte{0x7f, 0xff, 0xff, 0xff, 0xff, 0xff})
 54 | 	if len(rbuf) != 0 {
 55 | 		t.Errorf("len(rbuf)=%d", len(rbuf))
 56 | 	}
 57 | 	ans := uint64(1<<47-1)
 58 | 	if res != ans {
 59 | 		t.Errorf("got %d, expected %d", res, ans)
 60 | 	}
 61 | }
 62 | 
 63 | func TestBTree__readStInt64(t *testing.T) {
 64 | 	rbuf, res := readStInt64([]byte{0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff})
 65 | 	if len(rbuf) != 0 {
 66 | 		t.Errorf("len(rbuf)=%d", len(rbuf))
 67 | 	}
 68 | 	ans := int64(1<<63-1)
 69 | 	if res != ans {
 70 | 		t.Errorf("got %d, expected %d", res, ans)
 71 | 	}
 72 | }
 73 | 
 74 | func TestBTree__decodeRecord(t *testing.T) {
 75 | 	cases := []struct {
 76 | 		bt func() *btreeTable
 77 | 		payload []byte
 78 | 		match func (rec Record, err error) bool
 79 | 	}{
 80 | 		{
 81 | 			bt: func() *btreeTable{
 82 | 				return &btreeTable{}
 83 | 			},
 84 | 			payload: []byte{},
 85 | 			match: func (rec Record, err error) bool {
 86 | 				return err != nil
 87 | 			},
 88 | 		},
 89 | 		{
 90 | 
 91 | 			bt: func() *btreeTable {
 92 | 				db, _ := Open("testdata/firefox-history.sqlite")
 93 | 				page, _ := db.pager.Page(1)
 94 | 				bt, _ := newBtreeTable(page, db)
 95 | 				return bt
 96 | 			},
 97 | 			// From testdata/firefox-history.sqlite
 98 | 			payload: []byte{
 99 | 				6,23,75,37,1,0,105,110,100,101,120,115,113,108,105,116,101,95,97,
100 | 				117,116,111,105,110,100,101,120,95,109,111,122,95,107,101,121,119,
101 | 				111,114,100,115,95,49,109,111,122,95,107,101,121,119,111,114,100,
102 | 				115,26,
103 | 			},
104 | 			match: func(rec Record, err error) bool {
105 | 				body := []byte{
106 | 					105,110,100,101,120,115,113,108,105,116,101,95,97,117,116,111,
107 | 					105,110,100,101,120,95,109,111,122,95,107,101,121,119,111,114,
108 | 					100,115,95,49,109,111,122,95,107,101,121,119,111,114,100,115,26,
109 | 				}
110 | 				values:= []interface{}{"index", "sqlite_autoindex_moz_keywords_1", "moz_keywords", int8(26), nil}
111 | 				return rec.Header.Len == 5 && reflect.DeepEqual(rec.Values, values) && reflect.DeepEqual(rec.Body, body) && err == nil
112 | 			},
113 | 		},
114 | 	}
115 | 	for i := range cases {
116 | 		rec, err := cases[i].bt().decodeRecord(cases[i].payload)
117 | 		if !cases[i].match(rec, err) {
118 | 			t.Errorf("rec=%v\nerr=%v\n", rec, err)
119 | 		}
120 | 	}
121 | }
122 | 


--------------------------------------------------------------------------------
/cell.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2017 The go-sqlite Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | package sqlite3
 6 | 
 7 | // cellInfo holds information about an on-disk cell.
 8 | type cellInfo struct {
 9 | 	LeftChildPage int32
10 | 	RowID         *int64
11 | 	Payload       []byte
12 | 	OverflowPage  int32
13 | }
14 | 


--------------------------------------------------------------------------------
/cmd/sqlite-dump/main.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2018 The go-sqlite Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | // Command sqlite-dump is a simple command that dumps the high-level content
 6 | // of a SQLITE3 file on screen.
 7 | //
 8 | // Example:
 9 | //
10 | //  $> sqlite-dump ./testdata/test-1.sqlite
11 | //  sqlite3: opening "./testdata/test-1.sqlite"...
12 | //  sqlite3: version: 3008006
13 | //  sqlite3: page size: 1024
14 | //  sqlite3: num pages: 2
15 | //  sqlite3: num tables: 1
16 | //  sqlite3: === table[0] ===
17 | //  sqlite3: name: "tbl1"
18 | //  sqlite3: cols: 2
19 | //  sqlite3: col[0]: "one"
20 | //  sqlite3: col[1]: "two"
21 | package main
22 | 
23 | import (
24 | 	"flag"
25 | 	"fmt"
26 | 	"log"
27 | 	"os"
28 | 
29 | 	"github.com/go-sqlite/sqlite3"
30 | )
31 | 
32 | func main() {
33 | 	log.SetPrefix("sqlite3: ")
34 | 	log.SetFlags(0)
35 | 
36 | 	flag.Parse()
37 | 
38 | 	flag.Usage = func() {
39 | 		fmt.Fprintf(os.Stderr,
40 | 			`Usage: sqlite-dump [options] file1 [file2 [...]]
41 | 
42 | Ex:
43 | 
44 |  $> sqlite-dump ./testdata/test-1.sqlite
45 |  sqlite3: opening "./testdata/test-1.sqlite"...
46 |  sqlite3: version: 3008006
47 |  sqlite3: page size: 1024
48 |  sqlite3: num pages: 2
49 |  sqlite3: num tables: 1
50 |  sqlite3: === table[0] ===
51 |  sqlite3: name: "tbl1"
52 |  sqlite3: cols: 2
53 |  sqlite3: col[0]: "one"
54 |  sqlite3: col[1]: "two"
55 | 
56 | Options:
57 | `,
58 | 		)
59 | 	}
60 | 
61 | 	if flag.NArg() < 1 {
62 | 		flag.Usage()
63 | 		flag.PrintDefaults()
64 | 		os.Exit(1)
65 | 	}
66 | 
67 | 	for _, fname := range flag.Args() {
68 | 		process(fname)
69 | 	}
70 | }
71 | 
72 | func process(fname string) {
73 | 	log.Printf("opening %q...", fname)
74 | 	f, err := sqlite3.Open(fname)
75 | 	if err != nil {
76 | 		log.Printf("error opening %q: %v", fname, err)
77 | 		return
78 | 	}
79 | 	defer f.Close()
80 | 
81 | 	log.Printf("version: %v", f.Version())
82 | 	log.Printf("page size: %d", f.PageSize())
83 | 	log.Printf("num pages: %d", f.NumPage())
84 | 	log.Printf("num tables: %d", len(f.Tables()))
85 | 
86 | 	for i, table := range f.Tables() {
87 | 		log.Printf("=== table[%d] ===", i)
88 | 		log.Printf("name: %q", table.Name())
89 | 		log.Printf("cols: %d", len(table.Columns()))
90 | 		for j, col := range table.Columns() {
91 | 			log.Printf("col[%d]: %q", j, col.Name())
92 | 		}
93 | 	}
94 | }
95 | 


--------------------------------------------------------------------------------
/file.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2017 The go-sqlite Authors. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | package sqlite3
  6 | 
  7 | import (
  8 | 	"fmt"
  9 | 	"io"
 10 | 	"os"
 11 | 	"reflect"
 12 | 	"strings"
 13 | 
 14 | 	"github.com/gonuts/binary"
 15 | )
 16 | 
 17 | const (
 18 | 	sqlite3Magic = "SQLite format 3\x00"
 19 | 
 20 | 	printfDebug = false
 21 | )
 22 | 
 23 | var (
 24 | 	tblconstraints = []string{"CHECK", "FOREIGN KEY", "UNIQUE", "PRIMARY KEY"}
 25 | )
 26 | 
 27 | type DbFile struct {
 28 | 	pager  pager
 29 | 	header dbHeader
 30 | 	tables []Table
 31 | 	close  func() error
 32 | }
 33 | 
 34 | type dbHeader struct {
 35 | 	Magic         [16]byte
 36 | 	PageSize      uint16  // database page size in bytes
 37 | 	WVersion      byte    // file format write version
 38 | 	RVersion      byte    // file format read version
 39 | 	NReserved     byte    // bytes of unused reserved space at the end of each page
 40 | 	MaxFraction   byte    // maximum embedded payload fraction (must be 64)
 41 | 	MinFraction   byte    // minimum embedded payload fraction (must be 32)
 42 | 	LeafFraction  byte    // leaf payload fraction (must be 32)
 43 | 	NFileChanges  int32   // file change counter
 44 | 	DbSize        int32   // size of the database file in pages. The "in-header database size".
 45 | 	FreePage      int32   // page number of the first freelist trunk page.
 46 | 	NFreePages    int32   // total number of freelist pages.
 47 | 	SchemaCookie  [4]byte // schema cookie
 48 | 	SchemaFormat  int32   // schema format number. supported formats are 1,2,3 and 4.
 49 | 	PageCacheSize int32   // default page cache size
 50 | 	AutoVacuum    int32   // page number of the largest root b-tree page when in auto-vacuum or incremental-vacuum modes, or zero otherwise.
 51 | 
 52 | 	// the database text encoding.
 53 | 	//  1: UTF-8.
 54 | 	//  2: UTF-16le
 55 | 	//  3: UTF-16be
 56 | 	DbEncoding int32
 57 | 
 58 | 	UserVersion int32 // the "user version" as read and set by the user_version PRAGMA
 59 | 	IncrVacuum  int32 // tree (non-zero) for incremental-vacuum mode. False (zero) otherwise
 60 | 
 61 | 	ApplicationID int32 // the "Application ID" set by the PRAGMA application_id
 62 | 
 63 | 	XXX_reserved  [20]byte // reserved for expansion. must be zero
 64 | 	VersionValid  int32    // the version-valid-for number
 65 | 	SqliteVersion int32    // SQLITE_VERSION_NUMBER
 66 | }
 67 | 
 68 | func OpenFrom(f io.ReadSeeker) (*DbFile, error) {
 69 | 	var db DbFile
 70 | 
 71 | 	dec := binary.NewDecoder(f)
 72 | 	dec.Order = binary.BigEndian
 73 | 	err := dec.Decode(&db.header)
 74 | 	if err != nil {
 75 | 		return nil, err
 76 | 	}
 77 | 
 78 | 	if db.header.DbSize == 0 {
 79 | 		// determine it based on the size of the database file.
 80 | 		// if the size of the database file is not an integer multiple of
 81 | 		// the page-size, round down to the nearest page.
 82 | 		// except, any file larger than 0-bytes in size, is considered to
 83 | 		// contain at least one page.
 84 | 		size, err := f.Seek(0, io.SeekEnd)
 85 | 		if err != nil {
 86 | 			return nil, err
 87 | 		}
 88 | 		if _, err := f.Seek(0, io.SeekStart); err != nil {
 89 | 			return nil, err
 90 | 		}
 91 | 		pagesz := int64(db.header.PageSize)
 92 | 		npages := (size + pagesz - 1) / pagesz
 93 | 		db.header.DbSize = int32(npages)
 94 | 	}
 95 | 
 96 | 	if printfDebug {
 97 | 		fmt.Printf("db: %#v\n", db.header)
 98 | 	}
 99 | 	_, err = f.Seek(0, 0)
100 | 	if err != nil {
101 | 		return nil, err
102 | 	}
103 | 
104 | 	if string(db.header.Magic[:]) != sqlite3Magic {
105 | 		return nil, fmt.Errorf(
106 | 			"sqlite: invalid file header.\ngot:  %q\nwant: %q\n",
107 | 			string(db.header.Magic[:]),
108 | 			sqlite3Magic,
109 | 		)
110 | 	}
111 | 
112 | 	db.pager = newPager(f, db.PageSize(), db.NumPage())
113 | 
114 | 	err = db.init()
115 | 	if err != nil {
116 | 		return nil, err
117 | 	}
118 | 
119 | 	return &db, err
120 | }
121 | 
122 | func Open(fname string) (*DbFile, error) {
123 | 	f, err := os.Open(fname)
124 | 	if err != nil {
125 | 		return nil, err
126 | 	}
127 | 
128 | 	db, err := OpenFrom(f)
129 | 	if err != nil {
130 | 		f.Close()
131 | 		return nil, err
132 | 	}
133 | 	db.close = f.Close
134 | 	return db, nil
135 | }
136 | 
137 | func (db *DbFile) Close() error {
138 | 	db.pager.Delete()
139 | 	if db.close != nil {
140 | 		return db.close()
141 | 	}
142 | 	return nil
143 | }
144 | 
145 | // PageSize returns the database page size in bytes
146 | func (db *DbFile) PageSize() int {
147 | 	return int(db.header.PageSize)
148 | }
149 | 
150 | // NumPage returns the number of pages for this database
151 | func (db *DbFile) NumPage() int {
152 | 	return int(db.header.DbSize)
153 | }
154 | 
155 | // Encoding returns the text encoding for this database
156 | func (db *DbFile) Encoding() int {
157 | 	return int(db.header.DbEncoding)
158 | }
159 | 
160 | // Version returns the sqlite version number used to create this database
161 | func (db *DbFile) Version() int {
162 | 	return int(db.header.SqliteVersion)
163 | }
164 | 
165 | func (db *DbFile) Tables() []Table {
166 | 	return db.tables
167 | }
168 | 
169 | func (db *DbFile) init() error {
170 | 
171 | 	// load sqlite_master
172 | 	page, err := db.pager.Page(1)
173 | 	if err != nil {
174 | 		return err
175 | 	}
176 | 
177 | 	if page.Kind() != BTreeLeafTableKind && page.Kind() != BTreeInteriorTableKind {
178 | 		return fmt.Errorf("sqlite3: invalid page kind (%v)", page.Kind())
179 | 	}
180 | 
181 | 	btree, err := newBtreeTable(page, db)
182 | 	if err != nil {
183 | 		return err
184 | 	}
185 | 
186 | 	if printfDebug {
187 | 		fmt.Printf(">>> bt-hdr: %#v\n", btree.btheader)
188 | 		fmt.Printf(">>> init... (ncells=%d)\n", btree.NumCell())
189 | 	}
190 | 
191 | 	return btree.visitRecordsInorder(func(_ *int64, rec Record) error {
192 | 		// {"table", "tbl1", "tbl1", 2, "CREATE TABLE tbl1(one varchar(10), two smallint)"} (body=62)
193 | 		// {"table", "tbl2", "tbl2", 3, "CREATE TABLE tbl2(\n f1 varchar(30) primary key,\n f2 text,\n f3 real\n)"}
194 | 		if len(rec.Values) != 5 {
195 | 			return fmt.Errorf("sqlite3: invalid table format")
196 | 		}
197 | 
198 | 		rectype := rec.Values[0].(string)
199 | 		if rectype != "table" {
200 | 			return nil
201 | 		}
202 | 
203 | 		pageid := reflect.ValueOf(rec.Values[3])
204 | 		table := Table{
205 | 			name:   rec.Values[1].(string),
206 | 			pageid: int(pageid.Int()),
207 | 		}
208 | 
209 | 		// skip internal tables, aka don't expose them
210 | 		if strings.HasPrefix(table.name, "sqlite_") {
211 | 			return nil
212 | 		}
213 | 
214 | 		def := rec.Values[4].(string)
215 | 		def = strings.Replace(def, "CREATE TABLE "+table.name, "", 1)
216 | 		def = strings.Replace(def, "\n", "", -1)
217 | 		def = strings.TrimSpace(def)
218 | 		if def[0] == '(' {
219 | 			def = def[1:]
220 | 		}
221 | 		if def[len(def)-1] == ')' {
222 | 			def = def[:len(def)-1]
223 | 		}
224 | 		def = strings.TrimSpace(def)
225 | 
226 | 		parts := strings.Split(def, ",")
227 | 		// strip away statements like 'UNIQUE ...' or 'PRIMARY KEY ...' from a table definition
228 | 		for i := range parts {
229 | 			if i >= len(parts) {
230 | 				break // we removed at least one elem, so avoid out of bounds read
231 | 			}
232 | 
233 | 			parts[i] = strings.TrimSpace(parts[i])
234 | 			for j := range tblconstraints {
235 | 				if strings.HasPrefix(parts[i], tblconstraints[j]) {
236 | 					// drop all other elements
237 | 					parts = parts[:i]
238 | 					break
239 | 				}
240 | 			}
241 | 		}
242 | 
243 | 		table.cols = make([]Column, len(parts))
244 | 		for i := range parts {
245 | 			parts[i] = strings.TrimSpace(parts[i])
246 | 			idx := strings.Index(parts[i], " ") // find where col name ends and type starts
247 | 			if idx > 0 {
248 | 				table.cols[i].name = parts[i][:idx]
249 | 			} else {
250 | 				table.cols[i].name = parts[i]
251 | 			}
252 | 		}
253 | 
254 | 		if printfDebug {
255 | 			fmt.Printf(">>> def: %q => ncols=%d\n", def, len(table.cols))
256 | 		}
257 | 
258 | 		db.tables = append(db.tables, table)
259 | 		return nil
260 | 	})
261 | }
262 | 
263 | func (db *DbFile) Dumpdb() error {
264 | 	var err error
265 | 	for i := 1; i < db.NumPage(); i++ {
266 | 		page, err := db.pager.Page(i)
267 | 		if err != nil {
268 | 			fmt.Printf("error: sqlite3: error retrieving page-%d: %v\n", i, err)
269 | 			continue
270 | 		}
271 | 		fmt.Printf("page-%d: %v\n", i, page.Kind())
272 | 		btree, err := newBtreeTable(page, db)
273 | 		if err != nil {
274 | 			fmt.Printf("** error: %v\n", err)
275 | 			continue
276 | 		}
277 | 		for i := 0; i < btree.NumCell(); i++ {
278 | 			cell, err := btree.loadCell(i)
279 | 			if err != nil {
280 | 				fmt.Printf("** error: %v\n", err)
281 | 				continue
282 | 			}
283 | 			fmt.Printf("--- cell[%03d/%03d]= leftchildpage=%d row=%d payload=%d overflow=%d\n",
284 | 				i+1, btree.NumCell(),
285 | 				cell.LeftChildPage,
286 | 				cell.RowID,
287 | 				len(cell.Payload),
288 | 				cell.OverflowPage,
289 | 			)
290 | 		}
291 | 	}
292 | 
293 | 	return err
294 | }
295 | 
296 | // VisitTableRecords performs an inorder traversal of all cells in the
297 | // btree for the table with the given name, passing the (optional,
298 | // hence nullable) RowID, and record-decoded payload of each cell to
299 | // the visitor function `f`.
300 | func (db *DbFile) VisitTableRecords(tableName string, f func(*int64, Record) error) error {
301 | 	for _, table := range db.tables {
302 | 		if table.name != tableName {
303 | 			continue
304 | 		}
305 | 		page, err := db.pager.Page(table.pageid)
306 | 		if err != nil {
307 | 			return err
308 | 		}
309 | 		btree, err := newBtreeTable(page, db)
310 | 		if err != nil {
311 | 			return err
312 | 		}
313 | 		return btree.visitRecordsInorder(f)
314 | 	}
315 | 	return fmt.Errorf("unknown table %q", tableName)
316 | }
317 | 


--------------------------------------------------------------------------------
/page.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2017 The go-sqlite Authors. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | package sqlite3
  6 | 
  7 | import "fmt"
  8 | 
  9 | // PageKind describes what kind of page is.
 10 | type PageKind byte
 11 | 
 12 | /*
 13 | An index B-Tree internal node
 14 | An index B-Tree leaf node
 15 | A table B-Tree internal node
 16 | A table B-Tree leaf node
 17 | An overflow page
 18 | A freelist page
 19 | A pointer map page
 20 | The locking page
 21 | */
 22 | 
 23 | const (
 24 | 	intKeyKind   PageKind = 0x01
 25 | 	zeroDataKind PageKind = 0x02
 26 | 	leafDataKind PageKind = 0x04
 27 | 	leafKind     PageKind = 0x08
 28 | 
 29 | 	BTreeInteriorIndexKind = zeroDataKind
 30 | 	BTreeInteriorTableKind = leafDataKind | intKeyKind
 31 | 	BTreeLeafIndexKind     = zeroDataKind | leafKind
 32 | 	BTreeLeafTableKind     = leafDataKind | intKeyKind | leafKind
 33 | 
 34 | 	pkLockByte
 35 | 	pkFreelistTrunk
 36 | 	pkFreelistLeaf
 37 | 	pkPayloadOverflow
 38 | 	pkPointerMap
 39 | )
 40 | 
 41 | func (pk PageKind) String() string {
 42 | 	switch pk {
 43 | 	case BTreeInteriorIndexKind:
 44 | 		return "BTreeInteriorIndex"
 45 | 	case BTreeInteriorTableKind:
 46 | 		return "BTreeInteriorTable"
 47 | 	case BTreeLeafIndexKind:
 48 | 		return "BTreeLeafIndex"
 49 | 	case BTreeLeafTableKind:
 50 | 		return "BTreeLeafTable"
 51 | 	}
 52 | 
 53 | 	panic(fmt.Sprintf("sqlite3: invalid PageKind value (0x%02x)", byte(pk)))
 54 | }
 55 | 
 56 | /*
 57 | func newPage(i int, pb pageBuffer) (Page, error) {
 58 | 	pk := PageKind(pb.buf[0])
 59 | 	switch pk {
 60 | 	case BTreeInteriorIndexKind:
 61 | 		panic("not implemented")
 62 | 	case BTreeInteriorTableKind:
 63 | 		panic("not implemented")
 64 | 	case BTreeLeafIndexKind:
 65 | 		panic("not implemented")
 66 | 	case BTreeLeafTableKind:
 67 | 		return newBtreeLeafTable(i, pb)
 68 | 
 69 | 	}
 70 | 
 71 | 	panic(fmt.Errorf("invalid PageKind value (%d)", pk))
 72 | }
 73 | */
 74 | 
 75 | // page is a page loaded from disk.
 76 | type page struct {
 77 | 	id  int
 78 | 	pos int
 79 | 	buf []byte
 80 | }
 81 | 
 82 | func (p *page) ID() int {
 83 | 	return p.id
 84 | }
 85 | 
 86 | func (p *page) Kind() PageKind {
 87 | 	offset := 0
 88 | 	if p.id == 1 {
 89 | 		offset = 100
 90 | 	}
 91 | 	return PageKind(p.buf[0+offset])
 92 | }
 93 | 
 94 | func (p *page) PageSize() int {
 95 | 	return len(p.buf)
 96 | }
 97 | 
 98 | func (p *page) Seek(offset int64, whence int) (ret int64, err error) {
 99 | 	switch whence {
100 | 	case 0:
101 | 		offset := int(offset)
102 | 		if offset > len(p.buf) {
103 | 			return 0, fmt.Errorf("sqlite: offset too big (%d)", offset)
104 | 		}
105 | 		p.pos = offset
106 | 	case 1:
107 | 		offset := int(offset)
108 | 		pos := p.pos + offset
109 | 		if pos > len(p.buf) {
110 | 			return 0, fmt.Errorf("sqlite: offset too big (%d)", offset)
111 | 		}
112 | 		p.pos = pos
113 | 	case 2:
114 | 		offset := int(offset)
115 | 		pos := len(p.buf) - offset
116 | 		if pos < 0 {
117 | 			return 0, fmt.Errorf("sqlite: offset too big (%d)", offset)
118 | 		}
119 | 		p.pos = pos
120 | 	}
121 | 	return int64(p.pos), nil
122 | }
123 | 
124 | func (p *page) Pos() int {
125 | 	return p.pos
126 | }
127 | 
128 | func (p *page) Bytes() []byte {
129 | 	return p.buf[p.pos:]
130 | }
131 | 
132 | func (p *page) Decode(ptr interface{}) error {
133 | 	n, err := unmarshal(p.buf[p.pos:], ptr)
134 | 	if err != nil {
135 | 		return err
136 | 	}
137 | 	p.pos += int(n)
138 | 	return err
139 | }
140 | 
141 | func (p *page) Read(data []byte) (int, error) {
142 | 	n := copy(data, p.buf[p.pos:p.pos+len(data)])
143 | 	if n != len(data) {
144 | 		return n, fmt.Errorf("error. read too few bytes: %d. want %d", n, len(data))
145 | 	}
146 | 	p.pos += n
147 | 	return n, nil
148 | }
149 | 
150 | func (p *page) Varint() (int64, int) {
151 | 	v, n := varint(p.Bytes())
152 | 	if n <= 0 {
153 | 		return v, n
154 | 	}
155 | 	p.pos += int(n)
156 | 	return v, n
157 | }
158 | 


--------------------------------------------------------------------------------
/pager.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2017 The go-sqlite Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | package sqlite3
 6 | 
 7 | import (
 8 | 	"fmt"
 9 | 	"io"
10 | )
11 | 
12 | type pager struct {
13 | 	f      io.ReadSeeker
14 | 	size   int          // page size in bytes
15 | 	npages int          // total number of pages in db
16 | 	pages  map[int]page // cache of pages
17 | 	lru    []int        // list of last used pages
18 | }
19 | 
20 | func newPager(f io.ReadSeeker, size, npages int) pager {
21 | 	pager := pager{
22 | 		f:      f,
23 | 		size:   size,
24 | 		npages: npages,
25 | 		pages:  make(map[int]page, npages),
26 | 		lru:    make([]int, 0, 2),
27 | 	}
28 | 
29 | 	return pager
30 | }
31 | 
32 | func (p *pager) Page(i int) (page, error) {
33 | 	var err error
34 | 	page, ok := p.pages[i]
35 | 	if ok {
36 | 		return page, err
37 | 	}
38 | 
39 | 	if i > p.npages {
40 | 		return page, fmt.Errorf("sqlite3: out of range (%d > %d)", i, p.npages)
41 | 	}
42 | 
43 | 	pos, _ := p.f.Seek(0, io.SeekCurrent)
44 | 	defer p.f.Seek(pos, io.SeekStart)
45 | 
46 | 	buf := make([]byte, p.size)
47 | 	if _, err := p.f.Seek(int64((i-1)*p.size), io.SeekStart); err != nil {
48 | 		return page, err
49 | 	}
50 | 	n, err := p.f.Read(buf)
51 | 	if err != nil {
52 | 		return page, err
53 | 	}
54 | 
55 | 	if n != len(buf) {
56 | 		return page, fmt.Errorf("sqlite3: read too few bytes")
57 | 	}
58 | 
59 | 	page.id = i
60 | 	page.buf = buf
61 | 
62 | 	p.pages[i] = page
63 | 	p.lru = append(p.lru, i)
64 | 	return page, err
65 | }
66 | 
67 | func (p *pager) Delete() error {
68 | 	var err error
69 | 	p.pages = nil
70 | 	p.lru = nil
71 | 	return err
72 | }
73 | 


--------------------------------------------------------------------------------
/record.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2017 The go-sqlite Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | package sqlite3
 6 | 
 7 | type RecordHeader struct {
 8 | 	Len   int
 9 | 	Types []SerialType
10 | }
11 | 
12 | type Record struct {
13 | 	Header RecordHeader
14 | 	Body   []byte
15 | 	Values []interface{}
16 | }
17 | 


--------------------------------------------------------------------------------
/sqlite.go:
--------------------------------------------------------------------------------
1 | // Copyright 2017 The go-sqlite Authors. All rights reserved.
2 | // Use of this source code is governed by a BSD-style
3 | // license that can be found in the LICENSE file.
4 | 
5 | // Package sqlite3 decodes the SQLite-3 file format.
6 | package sqlite3 // import "github.com/go-sqlite/sqlite3"
7 | 


--------------------------------------------------------------------------------
/sqlite_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2017 The go-sqlite Authors. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | package sqlite3
  6 | 
  7 | import (
  8 | 	"bytes"
  9 | 	"fmt"
 10 | 	"os"
 11 | 	"testing"
 12 | 
 13 | 	"io/ioutil"
 14 | )
 15 | 
 16 | func TestFileOpen(t *testing.T) {
 17 | 
 18 | 	for _, test := range []struct {
 19 | 		fname   string
 20 | 		version int
 21 | 		npages  int
 22 | 		pagesz  int
 23 | 		tables  []Table
 24 | 		tblcount int
 25 | 	}{
 26 | 		{
 27 | 			fname:   "testdata/test-1.sqlite",
 28 | 			version: 3008006,
 29 | 			npages:  2,
 30 | 			pagesz:  1024,
 31 | 			tables: []Table{
 32 | 				Table{
 33 | 					name:   "tbl1",
 34 | 					pageid: 2,
 35 | 					cols: []Column{
 36 | 						Column{name: "one"},
 37 | 						Column{name: "two"},
 38 | 					},
 39 | 				},
 40 | 			},
 41 | 		},
 42 | 		{
 43 | 			fname:   "testdata/test-2.sqlite",
 44 | 			version: 3008006,
 45 | 			npages:  4,
 46 | 			pagesz:  1024,
 47 | 			tables: []Table{
 48 | 				Table{
 49 | 					name:   "tbl1",
 50 | 					pageid: 2,
 51 | 					cols: []Column{
 52 | 						Column{name: "one"},
 53 | 						Column{name: "two"},
 54 | 					},
 55 | 				},
 56 | 				Table{
 57 | 					name:   "tbl2",
 58 | 					pageid: 3,
 59 | 					cols: []Column{
 60 | 						Column{name: "f1"},
 61 | 						Column{name: "f2"},
 62 | 						Column{name: "f3"},
 63 | 					},
 64 | 				},
 65 | 			},
 66 | 		},
 67 | 		{
 68 | 			// Chrome history sqlite db
 69 | 			fname: "testdata/chrome-history.sqlite",
 70 | 			version: 3020000,
 71 | 			npages: 28,
 72 | 			pagesz: 4096,
 73 | 			tblcount: 11,
 74 | 		},
 75 | 		{
 76 | 			// Firefox history sqlite db
 77 | 			fname: "testdata/firefox-history.sqlite",
 78 | 			version: 3020001,
 79 | 			npages: 34,
 80 | 			pagesz: 32768,
 81 | 			tblcount: 10,
 82 | 		},
 83 | 		{
 84 | 			// Cover all integer types
 85 | 			fname:   "testdata/all-int-types.sqlite",
 86 | 			version: 3019003,
 87 | 			npages:  2,
 88 | 			pagesz:  4096,
 89 | 			tables: []Table{
 90 | 				Table{
 91 | 					name:   "ints",
 92 | 					pageid: 2,
 93 | 					cols: []Column{
 94 | 						Column{name: "i8"},
 95 | 						Column{name: "i16"},
 96 | 						Column{name: "i24"},
 97 | 						Column{name: "i32"},
 98 | 						Column{name: "i48"},
 99 | 						Column{name: "i64"},
100 | 					},
101 | 				},
102 | 			},
103 | 		},
104 | 		{
105 | 			// Partial Safari History.db
106 | 			fname: "testdata/safari-partial.sqlite",
107 | 			version: 3019003,
108 | 			npages: 3,
109 | 			pagesz: 4096,
110 | 			tables: []Table{
111 | 				Table{
112 | 					name: "metadata",
113 | 					pageid: 2,
114 | 					cols: []Column{
115 | 						Column{name: "key"},
116 | 						Column{name: "value"},
117 | 					},
118 | 				},
119 | 			},
120 | 		},
121 | 	} {
122 | 		t.Run(test.fname, func(t *testing.T) {
123 | 			f, err := Open(test.fname)
124 | 			if err != nil {
125 | 				t.Fatalf("could not open %s: %v", test.fname, err)
126 | 			}
127 | 			defer f.Close()
128 | 
129 | 			if f.Version() != test.version {
130 | 				t.Errorf("%s: version=%d. want=%d", test.fname, f.Version(), test.version)
131 | 			}
132 | 
133 | 			if f.PageSize() != test.pagesz {
134 | 				t.Errorf("%s: page size = %d. want=%d", test.fname, f.PageSize(), test.pagesz)
135 | 			}
136 | 
137 | 			if f.NumPage() != test.npages {
138 | 				t.Errorf("%s: num-pages = %d. want=%d", test.fname, f.NumPage(), test.npages)
139 | 			}
140 | 
141 | 			// Check tables
142 | 			if test.tblcount > 0 {
143 | 				if len(f.Tables()) == test.tblcount {
144 | 					t.Skip("parsed table size matches, but we aren't checking each table")
145 | 				}
146 | 				t.Errorf("%s: tables=%d, want=%d", test.fname, len(f.Tables()), test.tblcount)
147 | 			}
148 | 			// check each table
149 | 			if len(f.Tables()) != len(test.tables) {
150 | 				t.Errorf("%s: tables=%d, want=%d", test.fname, len(f.Tables()), len(test.tables))
151 | 			}
152 | 			n := len(f.Tables())
153 | 			if n > len(test.tables) {
154 | 				n = len(test.tables)
155 | 			}
156 | 			for i := 0; i < n; i++ {
157 | 				ftbl := f.Tables()[i]
158 | 				if ftbl.name != test.tables[i].name {
159 | 					t.Errorf("table name: got=%q, want=%q", ftbl.name, test.tables[i].name)
160 | 				}
161 | 				if ftbl.pageid != test.tables[i].pageid {
162 | 					t.Errorf("table pageid: got=%d, want=%d", ftbl.pageid, test.tables[i].pageid)
163 | 				}
164 | 				if len(ftbl.cols) != len(test.tables[i].cols) {
165 | 					t.Errorf("table %s cols: got=%v, want=%v", ftbl.name, ftbl.cols, test.tables[i].cols)
166 | 				}
167 | 				for j := range ftbl.cols {
168 | 					if ftbl.cols[j].name != test.tables[i].cols[j].name {
169 | 						t.Errorf("table %s column: got=%q, want=%q", ftbl.name, ftbl.cols[j].name, test.tables[i].cols[j].name)
170 | 					}
171 | 				}
172 | 			}
173 | 		})
174 | 
175 | 		name := fmt.Sprintf("%s:OpenFrom", test.fname)
176 | 		t.Run(name, func(t *testing.T) {
177 | 			d, err := os.Open(test.fname)
178 | 			if err != nil {
179 | 				t.Fatalf("could not open file %s: %v", test.fname, err)
180 | 			}
181 | 			defer d.Close()
182 | 			data, err := ioutil.ReadAll(d)
183 | 			if err != nil {
184 | 				t.Fatalf("could not read file %s: %v", test.fname, err)
185 | 			}
186 | 			r := bytes.NewReader(data)
187 | 
188 | 			f, err := OpenFrom(r)
189 | 			if err != nil {
190 | 				t.Fatalf("could not open db %s: %v", test.fname, err)
191 | 			}
192 | 			defer f.Close()
193 | 
194 | 			if f.Version() != test.version {
195 | 				t.Errorf("version=%d. want=%d", f.Version(), test.version)
196 | 			}
197 | 
198 | 			if f.PageSize() != test.pagesz {
199 | 				t.Errorf("page size = %d. want=%d", f.PageSize(), test.pagesz)
200 | 			}
201 | 
202 | 			if f.NumPage() != test.npages {
203 | 				t.Errorf("num-pages = %d. want=%d", f.NumPage(), test.npages)
204 | 			}
205 | 		})
206 | 	}
207 | 
208 | }
209 | 


--------------------------------------------------------------------------------
/stypes.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2017 The go-sqlite Authors. All rights reserved.
  2 | // Use of this source code is governed by a BSD-style
  3 | // license that can be found in the LICENSE file.
  4 | 
  5 | package sqlite3
  6 | 
  7 | import (
  8 | 	"fmt"
  9 | )
 10 | 
 11 | // SerialType represents SQLite types on disk
 12 | type SerialType int
 13 | 
 14 | const (
 15 | 	StNull SerialType = iota
 16 | 	StInt8
 17 | 	StInt16
 18 | 	StInt24
 19 | 	StInt32
 20 | 	StInt48
 21 | 	StInt64
 22 | 	StFloat
 23 | 	StC0
 24 | 	StC1
 25 | 
 26 | 	StBlob SerialType = 12
 27 | 	StText            = 13
 28 | )
 29 | 
 30 | func (st SerialType) String() string {
 31 | 	switch st {
 32 | 	case StNull:
 33 | 		return "StNull"
 34 | 	case StInt8:
 35 | 		return "StInt8"
 36 | 	case StInt16:
 37 | 		return "StInt16"
 38 | 	case StInt24:
 39 | 		return "StInt24"
 40 | 	case StInt32:
 41 | 		return "StInt32"
 42 | 	case StInt48:
 43 | 		return "StInt48"
 44 | 	case StInt64:
 45 | 		return "StInt64"
 46 | 	case StFloat:
 47 | 		return "StFloat"
 48 | 	case StC0:
 49 | 		return "StC0"
 50 | 	case StC1:
 51 | 		return "StC1"
 52 | 	}
 53 | 
 54 | 	if st.IsBlob() {
 55 | 		sz := int(st-12) / 2
 56 | 		return fmt.Sprintf("StBlob(%d)", sz)
 57 | 	}
 58 | 	if st.IsText() {
 59 | 		sz := int(st-13) / 2
 60 | 		return fmt.Sprintf("StText(%d)", sz)
 61 | 	}
 62 | 
 63 | 	panic("unreachable")
 64 | }
 65 | 
 66 | func (st SerialType) IsBlob() bool {
 67 | 	return st >= 12 && st&1 == 0
 68 | }
 69 | 
 70 | func (st SerialType) IsText() bool {
 71 | 	return st >= 13 && st&1 == 1
 72 | }
 73 | 
 74 | // NBytes returns the number of bytes on disk for this SerialType
 75 | // NBytes returns -1 if the SerialType is invalid.
 76 | func (st SerialType) NBytes() int {
 77 | 	switch st {
 78 | 	case StNull:
 79 | 		return 0
 80 | 	case StInt8:
 81 | 		return 1
 82 | 	case StInt16:
 83 | 		return 2
 84 | 	case StInt24:
 85 | 		return 3
 86 | 	case StInt32:
 87 | 		return 4
 88 | 	case StInt48:
 89 | 		return 6
 90 | 	case StInt64:
 91 | 		return 8
 92 | 	case StFloat:
 93 | 		return 8
 94 | 	case StC0, StC1:
 95 | 		return 0
 96 | 	}
 97 | 
 98 | 	if st.IsBlob() {
 99 | 		return int(st-12) / 2
100 | 	}
101 | 	if st.IsText() {
102 | 		return int(st-13) / 2
103 | 	}
104 | 
105 | 	return -1
106 | }
107 | 


--------------------------------------------------------------------------------
/table.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2017 The go-sqlite Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | package sqlite3
 6 | 
 7 | import (
 8 | 	"reflect"
 9 | )
10 | 
11 | // Table is a SQLite table
12 | type Table struct {
13 | 	name   string
14 | 	pageid int
15 | 	cols   []Column
16 | }
17 | 
18 | // Name returns the name of the table
19 | func (t *Table) Name() string {
20 | 	return t.name
21 | }
22 | 
23 | // NumRow returns the number of rows in the table
24 | func (t *Table) NumRow() int64 {
25 | 	//return t.nrows
26 | 	// FIXME(sbinet)
27 | 	return -1
28 | }
29 | 
30 | // Columns returns the columns of the table
31 | func (t *Table) Columns() []Column {
32 | 	return t.cols
33 | }
34 | 
35 | // Column describes a column in a SQLite table
36 | type Column struct {
37 | 	name string
38 | 	typ  reflect.Type
39 | }
40 | 
41 | // Name returns the name of the column
42 | func (col *Column) Name() string {
43 | 	return col.name
44 | }
45 | 
46 | // Type returns the SQLite type of the column
47 | func (col *Column) Type() reflect.Type {
48 | 	return col.typ
49 | }
50 | 


--------------------------------------------------------------------------------
/testdata/all-int-types.sqlite:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/go-sqlite/sqlite3/53dd8e640ee7dd6005bd7199eed0c470ab43a16e/testdata/all-int-types.sqlite


--------------------------------------------------------------------------------
/testdata/chrome-history.sqlite:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/go-sqlite/sqlite3/53dd8e640ee7dd6005bd7199eed0c470ab43a16e/testdata/chrome-history.sqlite


--------------------------------------------------------------------------------
/testdata/firefox-history.sqlite:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/go-sqlite/sqlite3/53dd8e640ee7dd6005bd7199eed0c470ab43a16e/testdata/firefox-history.sqlite


--------------------------------------------------------------------------------
/testdata/safari-partial.sqlite:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/go-sqlite/sqlite3/53dd8e640ee7dd6005bd7199eed0c470ab43a16e/testdata/safari-partial.sqlite


--------------------------------------------------------------------------------
/testdata/test-1.sqlite:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/go-sqlite/sqlite3/53dd8e640ee7dd6005bd7199eed0c470ab43a16e/testdata/test-1.sqlite


--------------------------------------------------------------------------------
/testdata/test-2.sqlite:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/go-sqlite/sqlite3/53dd8e640ee7dd6005bd7199eed0c470ab43a16e/testdata/test-2.sqlite


--------------------------------------------------------------------------------
/utils.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2017 The go-sqlite Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | package sqlite3
 6 | 
 7 | import (
 8 | 	"bytes"
 9 | 
10 | 	"github.com/gonuts/binary"
11 | )
12 | 
13 | func min(a, b int) int {
14 | 	if a > b {
15 | 		return b
16 | 	}
17 | 	return a
18 | }
19 | 
20 | func unmarshal(buf []byte, ptr interface{}) (int64, error) {
21 | 	r := bytes.NewReader(buf)
22 | 	max := r.Len()
23 | 	dec := binary.NewDecoder(r)
24 | 	dec.Order = binary.BigEndian
25 | 	err := dec.Decode(ptr)
26 | 	n := max - r.Len()
27 | 	return int64(n), err
28 | }
29 | 
30 | func varint(data []byte) (int64, int) {
31 | 	var val uint64
32 | 	for i := 0; i < 8; i++ {
33 | 		if i > len(data)-1 {
34 | 			return 0, 0
35 | 		}
36 | 		val = (val << 7) | uint64(data[i]&0x7f)
37 | 		if data[i] < 0x80 {
38 | 			return int64(val), i + 1
39 | 		}
40 | 	}
41 | 	if len(data) < 9 {
42 | 		return 0, 0
43 | 	}
44 | 	return int64((val << 8) | uint64(data[8])), 9
45 | }
46 | 


--------------------------------------------------------------------------------
/utils_test.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2017 The go-sqlite Authors. All rights reserved.
 2 | // Use of this source code is governed by a BSD-style
 3 | // license that can be found in the LICENSE file.
 4 | 
 5 | package sqlite3
 6 | 
 7 | import "testing"
 8 | 
 9 | func TestVarint(t *testing.T) {
10 | 	testdata := []struct {
11 | 		in      []byte
12 | 		valWant int64
13 | 		nWant   int
14 | 	}{
15 | 		{[]byte{0x7f}, 0x7f, 1},
16 | 		{[]byte{0x7f, 0x42}, 0x7f, 1},
17 | 		{[]byte{0x81}, 0x0, 0},
18 | 		{[]byte{0x81, 0x1}, 0x81, 2},
19 | 		{[]byte{0x83, 0x60}, 0x1e0, 2},
20 | 		{[]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, -1, 9},
21 | 		{[]byte{0x97, 0xa0, 0x80, 0xdf, 0xe9, 0xda, 0xf3, 0x02}, 13088612140104066, 8},
22 | 	}
23 | 
24 | 	for _, tt := range testdata {
25 | 		valGot, nGot := varint(tt.in)
26 | 
27 | 		if tt.valWant != valGot || tt.nWant != nGot {
28 | 			t.Errorf("want varint(%v) = (%d, %d); got (%d, %d)", tt.in, tt.valWant, tt.nWant, valGot, nGot)
29 | 		}
30 | 	}
31 | }
32 | 


--------------------------------------------------------------------------------