├── .github ├── ISSUE_TEMPLATE │ ├── config.yml │ └── open_an_issue.md ├── actions │ └── go-test-setup │ │ └── action.yml └── workflows │ ├── generated-pr.yml │ ├── go-check.yml │ ├── go-test.yml │ ├── release-check.yml │ ├── releaser.yml │ ├── stale.yml │ └── tagpush.yml ├── LICENSE ├── README.md ├── batching.go ├── ds_test.go ├── dstore.go ├── go.mod ├── go.sum ├── postgres └── postgres.go ├── sqlite ├── ds_test.go └── sqlite.go ├── txn.go └── version.json /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Getting Help on IPFS 4 | url: https://ipfs.io/help 5 | about: All information about how and where to get help on IPFS. 6 | - name: IPFS Official Forum 7 | url: https://discuss.ipfs.io 8 | about: Please post general questions, support requests, and discussions here. 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/open_an_issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Open an issue 3 | about: Only for actionable issues relevant to this repository. 4 | title: '' 5 | labels: need/triage 6 | assignees: '' 7 | 8 | --- 9 | 20 | -------------------------------------------------------------------------------- /.github/actions/go-test-setup/action.yml: -------------------------------------------------------------------------------- 1 | name: postgresql-and-timeout 2 | description: Start PostgreSQL and increase timeout 3 | 4 | runs: 5 | using: "composite" 6 | steps: 7 | - if: ${{ runner.os == 'Linux' }} 8 | run: | 9 | PGCONF="$(ls /etc/postgresql/*/main/pg_hba.conf)" 10 | sudo sed -i.bak -E "s/peer|scram-sha-256/trust/g" "$PGCONF" 11 | sudo service postgresql start 12 | shell: bash 13 | - if: ${{ runner.os == 'Windows' }} 14 | run: echo "$PGBIN" >> $GITHUB_PATH 15 | shell: bash 16 | - if: ${{ runner.os == 'macOS' }} 17 | run: echo "PGDATA=/usr/local/var/postgres" >> $GITHUB_ENV 18 | shell: bash 19 | - if: ${{ runner.os == 'macOS' }} 20 | run: pg_ctl -D "$PGDATA" initdb 21 | shell: bash 22 | - if: ${{ runner.os == 'Windows' || runner.os == 'macOS' }} 23 | run: pg_ctl -D "$PGDATA" -l "$PGDATA/server.log" start 24 | shell: bash 25 | - if: ${{ runner.os == 'macOS' }} 26 | run: createuser -s postgres 27 | shell: bash 28 | - run: echo "GOFLAGS=$GOFLAGS -timeout=30m" >> $GITHUB_ENV 29 | shell: bash 30 | -------------------------------------------------------------------------------- /.github/workflows/generated-pr.yml: -------------------------------------------------------------------------------- 1 | name: Close Generated PRs 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' 6 | workflow_dispatch: 7 | 8 | permissions: 9 | issues: write 10 | pull-requests: write 11 | 12 | jobs: 13 | stale: 14 | uses: ipdxco/unified-github-workflows/.github/workflows/reusable-generated-pr.yml@v1 15 | -------------------------------------------------------------------------------- /.github/workflows/go-check.yml: -------------------------------------------------------------------------------- 1 | name: Go Checks 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: ["master"] 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: read 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | go-check: 18 | uses: ipdxco/unified-github-workflows/.github/workflows/go-check.yml@v1.0 19 | -------------------------------------------------------------------------------- /.github/workflows/go-test.yml: -------------------------------------------------------------------------------- 1 | name: Go Test 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: ["master"] 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: read 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | go-test: 18 | uses: ipdxco/unified-github-workflows/.github/workflows/go-test.yml@v1.0 19 | secrets: 20 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 21 | -------------------------------------------------------------------------------- /.github/workflows/release-check.yml: -------------------------------------------------------------------------------- 1 | name: Release Checker 2 | 3 | on: 4 | pull_request_target: 5 | paths: [ 'version.json' ] 6 | types: [ opened, synchronize, reopened, labeled, unlabeled ] 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: write 11 | pull-requests: write 12 | 13 | concurrency: 14 | group: ${{ github.workflow }}-${{ github.ref }} 15 | cancel-in-progress: true 16 | 17 | jobs: 18 | release-check: 19 | uses: ipdxco/unified-github-workflows/.github/workflows/release-check.yml@v1.0 20 | -------------------------------------------------------------------------------- /.github/workflows/releaser.yml: -------------------------------------------------------------------------------- 1 | name: Releaser 2 | 3 | on: 4 | push: 5 | paths: [ 'version.json' ] 6 | workflow_dispatch: 7 | 8 | permissions: 9 | contents: write 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.sha }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | releaser: 17 | uses: ipdxco/unified-github-workflows/.github/workflows/releaser.yml@v1.0 18 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Close Stale Issues 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' 6 | workflow_dispatch: 7 | 8 | permissions: 9 | issues: write 10 | pull-requests: write 11 | 12 | jobs: 13 | stale: 14 | uses: ipdxco/unified-github-workflows/.github/workflows/reusable-stale-issue.yml@v1 15 | -------------------------------------------------------------------------------- /.github/workflows/tagpush.yml: -------------------------------------------------------------------------------- 1 | name: Tag Push Checker 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | permissions: 9 | contents: read 10 | issues: write 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | releaser: 18 | uses: ipdxco/unified-github-workflows/.github/workflows/tagpush.yml@v1.0 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Jeromy Johnson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SQL Datastore 2 | 3 | [![CircleCI](https://circleci.com/gh/ipfs/go-ds-sql.svg?style=shield)](https://circleci.com/gh/ipfs/go-ds-sql) 4 | [![Coverage](https://codecov.io/gh/ipfs/go-ds-sql/branch/master/graph/badge.svg)](https://codecov.io/gh/ipfs/go-ds-sql) 5 | [![Standard README](https://img.shields.io/badge/readme%20style-standard-brightgreen.svg)](https://github.com/RichardLitt/standard-readme) 6 | [![GoDoc](http://img.shields.io/badge/godoc-reference-5272B4.svg)](https://godoc.org/github.com/ipfs/go-ds-sql) 7 | [![golang version](https://img.shields.io/badge/golang-%3E%3D1.14.0-orange.svg)](https://golang.org/) 8 | [![Go Report Card](https://goreportcard.com/badge/github.com/ipfs/go-ds-sql)](https://goreportcard.com/report/github.com/ipfs/go-ds-sql) 9 | 10 | An implementation of [the datastore interface](https://github.com/ipfs/go-datastore) 11 | that can be backed by any sql database. 12 | 13 | ## Install 14 | 15 | ```sh 16 | go get github.com/ipfs/go-ds-sql 17 | ``` 18 | 19 | ## Usage 20 | 21 | ### PostgreSQL 22 | 23 | Ensure a database is created and a table exists with `key` and `data` columns. For example, in PostgreSQL you can create a table with the following structure (replacing `table_name` with the name of the table the datastore will use - by default this is `blocks`): 24 | 25 | ```sql 26 | CREATE TABLE IF NOT EXISTS table_name (key TEXT NOT NULL UNIQUE, data BYTEA) 27 | ``` 28 | 29 | It's recommended to create an index on the `key` column that is optimised for prefix scans. For example, in PostgreSQL you can create a `text_pattern_ops` index on the table: 30 | 31 | ```sql 32 | CREATE INDEX IF NOT EXISTS table_name_key_text_pattern_ops_idx ON table_name (key text_pattern_ops) 33 | ``` 34 | 35 | Import and use in your application: 36 | 37 | ```go 38 | import ( 39 | "database/sql" 40 | "github.com/ipfs/go-ds-sql" 41 | pg "github.com/ipfs/go-ds-sql/postgres" 42 | ) 43 | 44 | mydb, _ := sql.Open("yourdb", "yourdbparameters") 45 | 46 | // Implement the Queries interface for your SQL impl. 47 | // ...or use the provided PostgreSQL queries 48 | queries := pg.NewQueries("blocks") 49 | 50 | ds := sqlds.NewDatastore(mydb, queries) 51 | ``` 52 | 53 | ### SQLite 54 | 55 | The [SQLite](https://sqlite.org) wrapper tries to create the table automatically 56 | 57 | Prefix scans are optimized by using GLOB 58 | 59 | Import and use in your application: 60 | 61 | ```go 62 | package main 63 | 64 | import ( 65 | sqliteds "github.com/ipfs/go-ds-sql/sqlite" 66 | _ "github.com/mattn/go-sqlite3" 67 | ) 68 | 69 | func main() { 70 | opts := &sqliteds.Options{ 71 | DSN: "db.sqlite", 72 | } 73 | 74 | ds, err := opts.Create() 75 | if err != nil { 76 | panic(err) 77 | } 78 | defer func() { 79 | if err := ds.Close(); err != nil { 80 | panic(err) 81 | } 82 | }() 83 | } 84 | ``` 85 | 86 | If no `DSN` is specified, an unique in-memory database will be created 87 | 88 | ### SQLCipher 89 | 90 | The SQLite wrapper also supports the [SQLCipher](https://www.zetetic.net/sqlcipher/) extension 91 | 92 | Import and use in your application: 93 | 94 | ```go 95 | package main 96 | 97 | import ( 98 | sqliteds "github.com/ipfs/go-ds-sql/sqlite" 99 | _ "github.com/mutecomm/go-sqlcipher/v4" 100 | ) 101 | 102 | func main() { 103 | opts := &sqliteds.Options{ 104 | DSN: "encdb.sqlite", 105 | Key: ([]byte)("32_very_secure_bytes_0123456789a"), 106 | } 107 | 108 | ds, err := opts.Create() 109 | if err != nil { 110 | panic(err) 111 | } 112 | defer func() { 113 | if err := ds.Close(); err != nil { 114 | panic(err) 115 | } 116 | }() 117 | } 118 | ``` 119 | 120 | ## API 121 | 122 | [GoDoc Reference](https://godoc.org/github.com/ipfs/go-ds-sql) 123 | 124 | ## Contribute 125 | 126 | Feel free to dive in! [Open an issue](https://github.com/ipfs/go-ds-sql/issues/new) or submit PRs. 127 | 128 | ## License 129 | 130 | [MIT](LICENSE) 131 | -------------------------------------------------------------------------------- /batching.go: -------------------------------------------------------------------------------- 1 | package sqlds 2 | 3 | import ( 4 | "context" 5 | 6 | ds "github.com/ipfs/go-datastore" 7 | ) 8 | 9 | type op struct { 10 | delete bool 11 | value []byte 12 | } 13 | 14 | type batch struct { 15 | ds *Datastore 16 | ops map[ds.Key]op 17 | } 18 | 19 | // Batch creates a set of deferred updates to the database. 20 | // Since SQL does not support a true batch of updates, 21 | // operations are buffered and then executed sequentially 22 | // over a single connection when Commit is called. 23 | func (d *Datastore) Batch(ctx context.Context) (ds.Batch, error) { 24 | return &batch{ 25 | ds: d, 26 | ops: make(map[ds.Key]op), 27 | }, nil 28 | } 29 | 30 | func (bt *batch) Put(ctx context.Context, key ds.Key, val []byte) error { 31 | bt.ops[key] = op{value: val} 32 | return nil 33 | } 34 | 35 | func (bt *batch) Delete(ctx context.Context, key ds.Key) error { 36 | bt.ops[key] = op{delete: true} 37 | return nil 38 | } 39 | 40 | func (bt *batch) Commit(ctx context.Context) error { 41 | return bt.CommitContext(ctx) 42 | } 43 | 44 | func (bt *batch) CommitContext(ctx context.Context) error { 45 | conn, err := bt.ds.db.Conn(ctx) 46 | if err != nil { 47 | return err 48 | } 49 | defer conn.Close() 50 | 51 | for k, op := range bt.ops { 52 | if op.delete { 53 | _, err = conn.ExecContext(ctx, bt.ds.queries.Delete(), k.String()) 54 | } else { 55 | _, err = conn.ExecContext(ctx, bt.ds.queries.Put(), k.String(), op.value) 56 | } 57 | if err != nil { 58 | break 59 | } 60 | } 61 | 62 | return err 63 | } 64 | 65 | var _ ds.Batching = (*Datastore)(nil) 66 | -------------------------------------------------------------------------------- /ds_test.go: -------------------------------------------------------------------------------- 1 | package sqlds 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "crypto/rand" 7 | "database/sql" 8 | "fmt" 9 | "sort" 10 | "strings" 11 | "sync" 12 | "testing" 13 | 14 | ds "github.com/ipfs/go-datastore" 15 | dsq "github.com/ipfs/go-datastore/query" 16 | dstest "github.com/ipfs/go-datastore/test" 17 | _ "github.com/lib/pq" 18 | ) 19 | 20 | var initOnce sync.Once 21 | 22 | // Automatically re-create the test datastore. 23 | func initPG() { 24 | initOnce.Do(func() { 25 | fmtstr := "postgres://%s:%s@%s/?sslmode=disable" 26 | constr := fmt.Sprintf(fmtstr, "postgres", "", "127.0.0.1") 27 | db, err := sql.Open("postgres", constr) 28 | if err != nil { 29 | panic(err) 30 | } 31 | 32 | // drop/create the database. 33 | _, err = db.Exec("DROP DATABASE IF EXISTS test_datastore") 34 | if err != nil { 35 | panic(err) 36 | } 37 | _, err = db.Exec("CREATE DATABASE test_datastore") 38 | if err != nil { 39 | panic(err) 40 | } 41 | err = db.Close() 42 | if err != nil { 43 | panic(err) 44 | } 45 | }) 46 | } 47 | 48 | var testcases = map[string]string{ 49 | "/a": "a", 50 | "/a/b": "ab", 51 | "/a/b/c": "abc", 52 | "/a/b/d": "a/b/d", 53 | "/a/c": "ac", 54 | "/a/d": "ad", 55 | "/e": "e", 56 | "/f": "f", 57 | "/g": "", 58 | } 59 | 60 | type fakeQueries struct{} 61 | 62 | func (fakeQueries) Delete() string { 63 | return `DELETE FROM blocks WHERE key = $1` 64 | } 65 | 66 | func (fakeQueries) Exists() string { 67 | return `SELECT exists(SELECT 1 FROM blocks WHERE key=$1)` 68 | } 69 | 70 | func (fakeQueries) Get() string { 71 | return `SELECT data FROM blocks WHERE key = $1` 72 | } 73 | 74 | func (fakeQueries) Put() string { 75 | return `INSERT INTO blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET data = $2` 76 | } 77 | 78 | func (fakeQueries) Query() string { 79 | return `SELECT key, data FROM blocks` 80 | } 81 | 82 | func (fakeQueries) Prefix() string { 83 | return ` WHERE key LIKE '%s%%' ORDER BY key` 84 | } 85 | 86 | func (fakeQueries) Limit() string { 87 | return ` LIMIT %d` 88 | } 89 | 90 | func (fakeQueries) Offset() string { 91 | return ` OFFSET %d` 92 | } 93 | 94 | func (fakeQueries) GetSize() string { 95 | return `SELECT octet_length(data) FROM blocks WHERE key = $1` 96 | } 97 | 98 | // returns datastore, and a function to call on exit. 99 | // 100 | // d, close := newDS(t) 101 | // defer close() 102 | func newDS(t *testing.T) (*Datastore, func()) { 103 | initPG() 104 | // connect to that database. 105 | fmtstr := "postgres://%s:%s@%s/%s?sslmode=disable" 106 | constr := fmt.Sprintf(fmtstr, "postgres", "", "127.0.0.1", "test_datastore") 107 | db, err := sql.Open("postgres", constr) 108 | if err != nil { 109 | t.Fatal(err) 110 | } 111 | _, err = db.Exec("CREATE TABLE IF NOT EXISTS blocks (key TEXT NOT NULL UNIQUE, data BYTEA NOT NULL)") 112 | if err != nil { 113 | t.Fatal(err) 114 | } 115 | d := NewDatastore(db, fakeQueries{}) 116 | return d, func() { 117 | _, _ = d.db.Exec("DROP TABLE IF EXISTS blocks") 118 | d.Close() 119 | } 120 | } 121 | 122 | func addTestCases(t *testing.T, d *Datastore, testcases map[string]string) { 123 | for k, v := range testcases { 124 | dsk := ds.NewKey(k) 125 | if err := d.Put(context.Background(), dsk, []byte(v)); err != nil { 126 | t.Fatal(err) 127 | } 128 | } 129 | 130 | for k, v := range testcases { 131 | dsk := ds.NewKey(k) 132 | v2, err := d.Get(context.Background(), dsk) 133 | if err != nil { 134 | t.Fatal(err) 135 | } 136 | v2b := v2 137 | if string(v2b) != v { 138 | t.Errorf("%s values differ: %s != %s", k, v, v2) 139 | } 140 | } 141 | } 142 | 143 | func TestQuery(t *testing.T) { 144 | d, done := newDS(t) 145 | defer done() 146 | 147 | addTestCases(t, d, testcases) 148 | 149 | // test prefix 150 | rs, err := d.Query(context.Background(), dsq.Query{Prefix: "/a/"}) 151 | if err != nil { 152 | t.Fatal(err) 153 | } 154 | expectMatches(t, []string{ 155 | "/a/b", 156 | "/a/b/c", 157 | "/a/b/d", 158 | "/a/c", 159 | "/a/d", 160 | }, rs) 161 | 162 | // test offset and limit 163 | rs, err = d.Query(context.Background(), dsq.Query{Prefix: "/a/", Offset: 2, Limit: 2}) 164 | if err != nil { 165 | t.Fatal(err) 166 | } 167 | expectMatches(t, []string{ 168 | "/a/b/d", 169 | "/a/c", 170 | }, rs) 171 | 172 | // test orders 173 | orbk := dsq.OrderByKey{} 174 | orderByKey := []dsq.Order{orbk} 175 | rs, err = d.Query(context.Background(), dsq.Query{Prefix: "/a/", Orders: orderByKey}) 176 | if err != nil { 177 | t.Fatal(err) 178 | } 179 | expectKeyOrderMatches(t, rs, []string{ 180 | "/a/b", 181 | "/a/b/c", 182 | "/a/b/d", 183 | "/a/c", 184 | "/a/d", 185 | }) 186 | 187 | orbkd := dsq.OrderByKeyDescending{} 188 | orderByDesc := []dsq.Order{orbkd} 189 | rs, err = d.Query(context.Background(), dsq.Query{Prefix: "/a/", Orders: orderByDesc}) 190 | if err != nil { 191 | t.Fatal(err) 192 | } 193 | expectKeyOrderMatches(t, rs, []string{ 194 | "/a/d", 195 | "/a/c", 196 | "/a/b/d", 197 | "/a/b/c", 198 | "/a/b", 199 | }) 200 | 201 | // test filters 202 | equalFilter := dsq.FilterKeyCompare{Op: dsq.Equal, Key: "/a/b"} 203 | equalFilters := []dsq.Filter{equalFilter} 204 | rs, err = d.Query(context.Background(), dsq.Query{Prefix: "/a/", Filters: equalFilters}) 205 | if err != nil { 206 | t.Fatal(err) 207 | } 208 | expectKeyFilterMatches(t, rs, []string{"/a/b"}) 209 | 210 | greaterThanFilter := dsq.FilterKeyCompare{Op: dsq.GreaterThan, Key: "/a/b"} 211 | greaterThanFilters := []dsq.Filter{greaterThanFilter} 212 | rs, err = d.Query(context.Background(), dsq.Query{Prefix: "/a/", Filters: greaterThanFilters}) 213 | if err != nil { 214 | t.Fatal(err) 215 | } 216 | expectKeyFilterMatches(t, rs, []string{ 217 | "/a/b/c", 218 | "/a/b/d", 219 | "/a/c", 220 | "/a/d", 221 | }) 222 | 223 | lessThanFilter := dsq.FilterKeyCompare{Op: dsq.LessThanOrEqual, Key: "/a/b/c"} 224 | lessThanFilters := []dsq.Filter{lessThanFilter} 225 | rs, err = d.Query(context.Background(), dsq.Query{Prefix: "/a/", Filters: lessThanFilters}) 226 | if err != nil { 227 | t.Fatal(err) 228 | } 229 | expectKeyFilterMatches(t, rs, []string{ 230 | "/a/b", 231 | "/a/b/c", 232 | }) 233 | } 234 | 235 | func TestHas(t *testing.T) { 236 | d, done := newDS(t) 237 | defer done() 238 | addTestCases(t, d, testcases) 239 | 240 | has, err := d.Has(context.Background(), ds.NewKey("/a/b/c")) 241 | if err != nil { 242 | t.Error(err) 243 | } 244 | 245 | if !has { 246 | t.Error("Key should be found") 247 | } 248 | 249 | has, err = d.Has(context.Background(), ds.NewKey("/a/b/c/d")) 250 | if err != nil { 251 | t.Error(err) 252 | } 253 | 254 | if has { 255 | t.Error("Key should not be found") 256 | } 257 | } 258 | 259 | func TestNotExistGet(t *testing.T) { 260 | d, done := newDS(t) 261 | defer done() 262 | addTestCases(t, d, testcases) 263 | 264 | has, err := d.Has(context.Background(), ds.NewKey("/a/b/c/d")) 265 | if err != nil { 266 | t.Error(err) 267 | } 268 | 269 | if has { 270 | t.Error("Key should not be found") 271 | } 272 | 273 | val, err := d.Get(context.Background(), ds.NewKey("/a/b/c/d")) 274 | if val != nil { 275 | t.Error("Key should not be found") 276 | } 277 | 278 | if err != ds.ErrNotFound { 279 | t.Error("Error was not set to ds.ErrNotFound") 280 | if err != nil { 281 | t.Error(err) 282 | } 283 | } 284 | } 285 | 286 | func TestDelete(t *testing.T) { 287 | d, done := newDS(t) 288 | defer done() 289 | addTestCases(t, d, testcases) 290 | 291 | has, err := d.Has(context.Background(), ds.NewKey("/a/b/c")) 292 | if err != nil { 293 | t.Error(err) 294 | } 295 | if !has { 296 | t.Error("Key should be found") 297 | } 298 | 299 | err = d.Delete(context.Background(), ds.NewKey("/a/b/c")) 300 | if err != nil { 301 | t.Error(err) 302 | } 303 | 304 | has, err = d.Has(context.Background(), ds.NewKey("/a/b/c")) 305 | if err != nil { 306 | t.Error(err) 307 | } 308 | if has { 309 | t.Error("Key should not be found") 310 | } 311 | } 312 | 313 | func TestGetEmpty(t *testing.T) { 314 | d, done := newDS(t) 315 | defer done() 316 | 317 | err := d.Put(context.Background(), ds.NewKey("/a"), []byte{}) 318 | if err != nil { 319 | t.Error(err) 320 | } 321 | 322 | v, err := d.Get(context.Background(), ds.NewKey("/a")) 323 | if err != nil { 324 | t.Error(err) 325 | } 326 | 327 | if len(v) != 0 { 328 | t.Error("expected 0 len []byte form get") 329 | } 330 | } 331 | 332 | func TestBatching(t *testing.T) { 333 | d, done := newDS(t) 334 | defer done() 335 | 336 | b, err := d.Batch(context.Background()) 337 | if err != nil { 338 | t.Fatal(err) 339 | } 340 | 341 | for k, v := range testcases { 342 | err := b.Put(context.Background(), ds.NewKey(k), []byte(v)) 343 | if err != nil { 344 | t.Fatal(err) 345 | } 346 | } 347 | 348 | err = b.Commit(context.Background()) 349 | if err != nil { 350 | t.Fatal(err) 351 | } 352 | 353 | for k, v := range testcases { 354 | val, err := d.Get(context.Background(), ds.NewKey(k)) 355 | if err != nil { 356 | t.Fatal(err) 357 | } 358 | 359 | if v != string(val) { 360 | t.Fatal("got wrong data!") 361 | } 362 | } 363 | 364 | //Test delete 365 | b, err = d.Batch(context.Background()) 366 | if err != nil { 367 | t.Fatal(err) 368 | } 369 | 370 | err = b.Delete(context.Background(), ds.NewKey("/a/b")) 371 | if err != nil { 372 | t.Fatal(err) 373 | } 374 | 375 | err = b.Delete(context.Background(), ds.NewKey("/a/b/c")) 376 | if err != nil { 377 | t.Fatal(err) 378 | } 379 | 380 | err = b.Commit(context.Background()) 381 | if err != nil { 382 | t.Fatal(err) 383 | } 384 | 385 | rs, err := d.Query(context.Background(), dsq.Query{Prefix: "/"}) 386 | if err != nil { 387 | t.Fatal(err) 388 | } 389 | 390 | expectMatches(t, []string{ 391 | "/a", 392 | "/a/b/d", 393 | "/a/c", 394 | "/a/d", 395 | "/e", 396 | "/f", 397 | "/g", 398 | }, rs) 399 | } 400 | 401 | func SubtestBasicPutGet(t *testing.T) { 402 | d, done := newDS(t) 403 | defer done() 404 | 405 | k := ds.NewKey("foo") 406 | val := []byte("Hello Datastore!") 407 | 408 | err := d.Put(context.Background(), k, val) 409 | if err != nil { 410 | t.Fatal("error putting to datastore: ", err) 411 | } 412 | 413 | have, err := d.Has(context.Background(), k) 414 | if err != nil { 415 | t.Fatal("error calling has on key we just put: ", err) 416 | } 417 | 418 | if !have { 419 | t.Fatal("should have key foo, has returned false") 420 | } 421 | 422 | size, err := d.GetSize(context.Background(), k) 423 | if err != nil { 424 | t.Fatal("error getting size after put: ", err) 425 | } 426 | if size != len(val) { 427 | t.Fatalf("incorrect size: expected %d, got %d", len(val), size) 428 | } 429 | 430 | out, err := d.Get(context.Background(), k) 431 | if err != nil { 432 | t.Fatal("error getting value after put: ", err) 433 | } 434 | 435 | if !bytes.Equal(out, val) { 436 | t.Fatal("value received on get wasnt what we expected:", out) 437 | } 438 | 439 | have, err = d.Has(context.Background(), k) 440 | if err != nil { 441 | t.Fatal("error calling has after get: ", err) 442 | } 443 | 444 | if !have { 445 | t.Fatal("should have key foo, has returned false") 446 | } 447 | 448 | size, err = d.GetSize(context.Background(), k) 449 | if err != nil { 450 | t.Fatal("error getting size after get: ", err) 451 | } 452 | if size != len(val) { 453 | t.Fatalf("incorrect size: expected %d, got %d", len(val), size) 454 | } 455 | 456 | err = d.Delete(context.Background(), k) 457 | if err != nil { 458 | t.Fatal("error calling delete: ", err) 459 | } 460 | 461 | have, err = d.Has(context.Background(), k) 462 | if err != nil { 463 | t.Fatal("error calling has after delete: ", err) 464 | } 465 | 466 | if have { 467 | t.Fatal("should not have key foo, has returned true") 468 | } 469 | 470 | size, err = d.GetSize(context.Background(), k) 471 | switch err { 472 | case ds.ErrNotFound: 473 | case nil: 474 | t.Fatal("expected error getting size after delete") 475 | default: 476 | t.Fatal("wrong error getting size after delete: ", err) 477 | } 478 | if size != -1 { 479 | t.Fatal("expected missing size to be -1") 480 | } 481 | } 482 | 483 | func TestNotFounds(t *testing.T) { 484 | d, done := newDS(t) 485 | defer done() 486 | 487 | badk := ds.NewKey("notreal") 488 | 489 | val, err := d.Get(context.Background(), badk) 490 | if err != ds.ErrNotFound { 491 | t.Fatal("expected ErrNotFound for key that doesnt exist, got: ", err) 492 | } 493 | 494 | if val != nil { 495 | t.Fatal("get should always return nil for not found values") 496 | } 497 | 498 | have, err := d.Has(context.Background(), badk) 499 | if err != nil { 500 | t.Fatal("error calling has on not found key: ", err) 501 | } 502 | if have { 503 | t.Fatal("has returned true for key we don't have") 504 | } 505 | 506 | size, err := d.GetSize(context.Background(), badk) 507 | switch err { 508 | case ds.ErrNotFound: 509 | case nil: 510 | t.Fatal("expected error getting size after delete") 511 | default: 512 | t.Fatal("wrong error getting size after delete: ", err) 513 | } 514 | if size != -1 { 515 | t.Fatal("expected missing size to be -1") 516 | } 517 | } 518 | 519 | func SubtestManyKeysAndQuery(t *testing.T) { 520 | d, done := newDS(t) 521 | defer done() 522 | 523 | var keys []ds.Key 524 | var keystrs []string 525 | var values [][]byte 526 | count := 100 527 | for i := 0; i < count; i++ { 528 | s := fmt.Sprintf("%dkey%d", i, i) 529 | dsk := ds.NewKey(s) 530 | keystrs = append(keystrs, dsk.String()) 531 | keys = append(keys, dsk) 532 | buf := make([]byte, 64) 533 | _, _ = rand.Read(buf) 534 | values = append(values, buf) 535 | } 536 | 537 | t.Logf("putting %d values", count) 538 | for i, k := range keys { 539 | err := d.Put(context.Background(), k, values[i]) 540 | if err != nil { 541 | t.Fatalf("error on put[%d]: %s", i, err) 542 | } 543 | } 544 | 545 | t.Log("getting values back") 546 | for i, k := range keys { 547 | val, err := d.Get(context.Background(), k) 548 | if err != nil { 549 | t.Fatalf("error on get[%d]: %s", i, err) 550 | } 551 | 552 | if !bytes.Equal(val, values[i]) { 553 | t.Fatal("input value didnt match the one returned from Get") 554 | } 555 | } 556 | 557 | t.Log("querying values") 558 | q := dsq.Query{KeysOnly: true} 559 | resp, err := d.Query(context.Background(), q) 560 | if err != nil { 561 | t.Fatal("calling query: ", err) 562 | } 563 | 564 | t.Log("aggregating query results") 565 | var outkeys []string 566 | for { 567 | res, ok := resp.NextSync() 568 | if res.Error != nil { 569 | t.Fatal("query result error: ", res.Error) 570 | } 571 | if !ok { 572 | break 573 | } 574 | 575 | outkeys = append(outkeys, res.Key) 576 | } 577 | 578 | t.Log("verifying query output") 579 | sort.Strings(keystrs) 580 | sort.Strings(outkeys) 581 | 582 | if len(keystrs) != len(outkeys) { 583 | t.Fatal("got wrong number of keys back") 584 | } 585 | 586 | for i, s := range keystrs { 587 | if outkeys[i] != s { 588 | t.Fatalf("in key output, got %s but expected %s", outkeys[i], s) 589 | } 590 | } 591 | 592 | t.Log("deleting all keys") 593 | for _, k := range keys { 594 | if err := d.Delete(context.Background(), k); err != nil { 595 | t.Fatal(err) 596 | } 597 | } 598 | } 599 | 600 | // Tests from basic_tests from go-datastore 601 | func TestBasicPutGet(t *testing.T) { 602 | d, done := newDS(t) 603 | defer done() 604 | 605 | k := ds.NewKey("foo") 606 | val := []byte("Hello Datastore!") 607 | 608 | err := d.Put(context.Background(), k, val) 609 | if err != nil { 610 | t.Fatal("error putting to datastore: ", err) 611 | } 612 | 613 | have, err := d.Has(context.Background(), k) 614 | if err != nil { 615 | t.Fatal("error calling has on key we just put: ", err) 616 | } 617 | 618 | if !have { 619 | t.Fatal("should have key foo, has returned false") 620 | } 621 | 622 | out, err := d.Get(context.Background(), k) 623 | if err != nil { 624 | t.Fatal("error getting value after put: ", err) 625 | } 626 | 627 | if !bytes.Equal(out, val) { 628 | t.Fatal("value received on get wasnt what we expected:", out) 629 | } 630 | 631 | have, err = d.Has(context.Background(), k) 632 | if err != nil { 633 | t.Fatal("error calling has after get: ", err) 634 | } 635 | 636 | if !have { 637 | t.Fatal("should have key foo, has returned false") 638 | } 639 | 640 | err = d.Delete(context.Background(), k) 641 | if err != nil { 642 | t.Fatal("error calling delete: ", err) 643 | } 644 | 645 | have, err = d.Has(context.Background(), k) 646 | if err != nil { 647 | t.Fatal("error calling has after delete: ", err) 648 | } 649 | 650 | if have { 651 | t.Fatal("should not have key foo, has returned true") 652 | } 653 | SubtestBasicPutGet(t) 654 | } 655 | 656 | func TestManyKeysAndQuery(t *testing.T) { 657 | d, done := newDS(t) 658 | defer done() 659 | 660 | var keys []ds.Key 661 | var keystrs []string 662 | var values [][]byte 663 | count := 100 664 | for i := 0; i < count; i++ { 665 | s := fmt.Sprintf("%dkey%d", i, i) 666 | dsk := ds.NewKey(s) 667 | keystrs = append(keystrs, dsk.String()) 668 | keys = append(keys, dsk) 669 | buf := make([]byte, 64) 670 | _, _ = rand.Read(buf) 671 | values = append(values, buf) 672 | } 673 | 674 | t.Logf("putting %d values", count) 675 | for i, k := range keys { 676 | err := d.Put(context.Background(), k, values[i]) 677 | if err != nil { 678 | t.Fatalf("error on put[%d]: %s", i, err) 679 | } 680 | } 681 | 682 | t.Log("getting values back") 683 | for i, k := range keys { 684 | val, err := d.Get(context.Background(), k) 685 | if err != nil { 686 | t.Fatalf("error on get[%d]: %s", i, err) 687 | } 688 | 689 | if !bytes.Equal(val, values[i]) { 690 | t.Fatal("input value didnt match the one returned from Get") 691 | } 692 | } 693 | 694 | t.Log("querying values") 695 | q := dsq.Query{KeysOnly: true} 696 | resp, err := d.Query(context.Background(), q) 697 | if err != nil { 698 | t.Fatal("calling query: ", err) 699 | } 700 | 701 | t.Log("aggregating query results") 702 | var outkeys []string 703 | for { 704 | res, ok := resp.NextSync() 705 | if res.Error != nil { 706 | t.Fatal("query result error: ", res.Error) 707 | } 708 | if !ok { 709 | break 710 | } 711 | 712 | outkeys = append(outkeys, res.Key) 713 | } 714 | 715 | t.Log("verifying query output") 716 | sort.Strings(keystrs) 717 | sort.Strings(outkeys) 718 | 719 | if len(keystrs) != len(outkeys) { 720 | t.Fatalf("got wrong number of keys back, %d != %d", len(keystrs), len(outkeys)) 721 | } 722 | 723 | for i, s := range keystrs { 724 | if outkeys[i] != s { 725 | t.Fatalf("in key output, got %s but expected %s", outkeys[i], s) 726 | } 727 | } 728 | 729 | t.Log("deleting all keys") 730 | for _, k := range keys { 731 | if err := d.Delete(context.Background(), k); err != nil { 732 | t.Fatal(err) 733 | } 734 | } 735 | 736 | SubtestManyKeysAndQuery(t) 737 | } 738 | 739 | func TestSuite(t *testing.T) { 740 | d, done := newDS(t) 741 | defer done() 742 | 743 | dstest.SubtestAll(t, d) 744 | } 745 | 746 | func expectMatches(t *testing.T, expect []string, actualR dsq.Results) { 747 | t.Helper() 748 | actual, err := actualR.Rest() 749 | if err != nil { 750 | t.Error(err) 751 | } 752 | 753 | if len(actual) != len(expect) { 754 | t.Error("not enough", expect, actual) 755 | } 756 | for _, k := range expect { 757 | found := false 758 | for _, e := range actual { 759 | if e.Key == k { 760 | found = true 761 | } 762 | } 763 | if !found { 764 | t.Error(k, "not found") 765 | } 766 | } 767 | } 768 | 769 | func expectKeyOrderMatches(t *testing.T, actual dsq.Results, expect []string) { 770 | rs, err := actual.Rest() 771 | if err != nil { 772 | t.Error("error fetching dsq.Results", expect, actual) 773 | return 774 | } 775 | 776 | if len(rs) != len(expect) { 777 | t.Error("expect != actual.", expect, actual) 778 | return 779 | } 780 | 781 | for i, r := range rs { 782 | if r.Key != expect[i] { 783 | t.Error("expect != actual.", expect, actual) 784 | return 785 | } 786 | } 787 | } 788 | 789 | func expectKeyFilterMatches(t *testing.T, actual dsq.Results, expect []string) { 790 | actualE, err := actual.Rest() 791 | if err != nil { 792 | t.Error(err) 793 | return 794 | } 795 | actualS := make([]string, len(actualE)) 796 | for i, e := range actualE { 797 | actualS[i] = e.Key 798 | } 799 | 800 | if len(actualS) != len(expect) { 801 | t.Error("length doesn't match.", expect, actualS) 802 | return 803 | } 804 | 805 | if strings.Join(actualS, "") != strings.Join(expect, "") { 806 | t.Error("expect != actual.", expect, actualS) 807 | return 808 | } 809 | } 810 | -------------------------------------------------------------------------------- /dstore.go: -------------------------------------------------------------------------------- 1 | package sqlds 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | 8 | ds "github.com/ipfs/go-datastore" 9 | dsq "github.com/ipfs/go-datastore/query" 10 | ) 11 | 12 | // Queries generates SQL queries for datastore operations. 13 | type Queries interface { 14 | Delete() string 15 | Exists() string 16 | Get() string 17 | Put() string 18 | Query() string 19 | Prefix() string 20 | Limit() string 21 | Offset() string 22 | GetSize() string 23 | } 24 | 25 | // Datastore is a SQL backed datastore. 26 | type Datastore struct { 27 | db *sql.DB 28 | queries Queries 29 | } 30 | 31 | // NewDatastore returns a new SQL datastore. 32 | func NewDatastore(db *sql.DB, queries Queries) *Datastore { 33 | return &Datastore{db: db, queries: queries} 34 | } 35 | 36 | // Close closes the underying SQL database. 37 | func (d *Datastore) Close() error { 38 | return d.db.Close() 39 | } 40 | 41 | // Delete removes a row from the SQL database by the given key. 42 | func (d *Datastore) Delete(ctx context.Context, key ds.Key) error { 43 | _, err := d.db.ExecContext(ctx, d.queries.Delete(), key.String()) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | return nil 49 | } 50 | 51 | // Get retrieves a value from the SQL database by the given key. 52 | func (d *Datastore) Get(ctx context.Context, key ds.Key) (value []byte, err error) { 53 | row := d.db.QueryRowContext(ctx, d.queries.Get(), key.String()) 54 | var out []byte 55 | 56 | switch err := row.Scan(&out); err { 57 | case sql.ErrNoRows: 58 | return nil, ds.ErrNotFound 59 | case nil: 60 | return out, nil 61 | default: 62 | return nil, err 63 | } 64 | } 65 | 66 | // Has determines if a value for the given key exists in the SQL database. 67 | func (d *Datastore) Has(ctx context.Context, key ds.Key) (exists bool, err error) { 68 | row := d.db.QueryRowContext(ctx, d.queries.Exists(), key.String()) 69 | 70 | switch err := row.Scan(&exists); err { 71 | case sql.ErrNoRows: 72 | return exists, nil 73 | case nil: 74 | return exists, nil 75 | default: 76 | return exists, err 77 | } 78 | } 79 | 80 | // Put "upserts" a row into the SQL database. 81 | func (d *Datastore) Put(ctx context.Context, key ds.Key, value []byte) error { 82 | _, err := d.db.ExecContext(ctx, d.queries.Put(), key.String(), value) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | return nil 88 | } 89 | 90 | // Query returns multiple rows from the SQL database based on the passed query parameters. 91 | func (d *Datastore) Query(ctx context.Context, q dsq.Query) (dsq.Results, error) { 92 | raw, err := d.rawQuery(ctx, q) 93 | if err != nil { 94 | return nil, err 95 | } 96 | 97 | for _, f := range q.Filters { 98 | raw = dsq.NaiveFilter(raw, f) 99 | } 100 | 101 | raw = dsq.NaiveOrder(raw, q.Orders...) 102 | 103 | // if we have filters or orders, offset and limit won't have been applied in the query 104 | if len(q.Filters) > 0 || len(q.Orders) > 0 { 105 | if q.Offset != 0 { 106 | raw = dsq.NaiveOffset(raw, q.Offset) 107 | } 108 | if q.Limit != 0 { 109 | raw = dsq.NaiveLimit(raw, q.Limit) 110 | } 111 | } 112 | 113 | return raw, nil 114 | } 115 | 116 | func (d *Datastore) rawQuery(ctx context.Context, q dsq.Query) (dsq.Results, error) { 117 | var rows *sql.Rows 118 | var err error 119 | 120 | rows, err = queryWithParams(ctx, d, q) 121 | if err != nil { 122 | return nil, err 123 | } 124 | 125 | it := dsq.Iterator{ 126 | Next: func() (dsq.Result, bool) { 127 | if !rows.Next() { 128 | return dsq.Result{}, false 129 | } 130 | 131 | var key string 132 | var out []byte 133 | 134 | err := rows.Scan(&key, &out) 135 | if err != nil { 136 | return dsq.Result{Error: err}, false 137 | } 138 | 139 | entry := dsq.Entry{Key: key} 140 | 141 | if !q.KeysOnly { 142 | entry.Value = out 143 | } 144 | if q.ReturnsSizes { 145 | entry.Size = len(out) 146 | } 147 | 148 | return dsq.Result{Entry: entry}, true 149 | }, 150 | Close: func() error { 151 | return nil 152 | }, 153 | } 154 | 155 | return dsq.ResultsFromIterator(q, it), nil 156 | } 157 | 158 | // Sync is noop for SQL databases. 159 | func (d *Datastore) Sync(ctx context.Context, key ds.Key) error { 160 | return nil 161 | } 162 | 163 | // GetSize determines the size in bytes of the value for a given key. 164 | func (d *Datastore) GetSize(ctx context.Context, key ds.Key) (int, error) { 165 | row := d.db.QueryRowContext(ctx, d.queries.GetSize(), key.String()) 166 | var size int 167 | 168 | switch err := row.Scan(&size); err { 169 | case sql.ErrNoRows: 170 | return -1, ds.ErrNotFound 171 | case nil: 172 | return size, nil 173 | default: 174 | return 0, err 175 | } 176 | } 177 | 178 | // queryWithParams applies prefix, limit, and offset params in pg query 179 | func queryWithParams(ctx context.Context, d *Datastore, q dsq.Query) (*sql.Rows, error) { 180 | var qNew = d.queries.Query() 181 | 182 | if q.Prefix != "" { 183 | // normalize 184 | prefix := ds.NewKey(q.Prefix).String() 185 | if prefix != "/" { 186 | qNew += fmt.Sprintf(d.queries.Prefix(), prefix+"/") 187 | } 188 | } 189 | 190 | // only apply limit and offset if we do not have to naive filter/order the results 191 | if len(q.Filters) == 0 && len(q.Orders) == 0 { 192 | if q.Limit != 0 { 193 | qNew += fmt.Sprintf(d.queries.Limit(), q.Limit) 194 | } 195 | if q.Offset != 0 { 196 | qNew += fmt.Sprintf(d.queries.Offset(), q.Offset) 197 | } 198 | } 199 | 200 | return d.db.QueryContext(ctx, qNew) 201 | 202 | } 203 | 204 | var _ ds.Datastore = (*Datastore)(nil) 205 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ipfs/go-ds-sql 2 | 3 | go 1.23 4 | 5 | require ( 6 | github.com/ipfs/go-datastore v0.8.2 7 | github.com/lib/pq v1.10.9 8 | github.com/mattn/go-sqlite3 v1.14.24 9 | ) 10 | 11 | require ( 12 | github.com/google/uuid v1.6.0 // indirect 13 | github.com/ipfs/go-detect-race v0.0.1 // indirect 14 | ) 15 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 2 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 3 | github.com/ipfs/go-datastore v0.8.2 h1:Jy3wjqQR6sg/LhyY0NIePZC3Vux19nLtg7dx0TVqr6U= 4 | github.com/ipfs/go-datastore v0.8.2/go.mod h1:W+pI1NsUsz3tcsAACMtfC+IZdnQTnC/7VfPoJBQuts0= 5 | github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= 6 | github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= 7 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 8 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 9 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 10 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 11 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 12 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 13 | github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= 14 | github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= 15 | github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= 16 | github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= 17 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 18 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 19 | -------------------------------------------------------------------------------- /postgres/postgres.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | 7 | sqlds "github.com/ipfs/go-ds-sql" 8 | 9 | _ "github.com/lib/pq" //postgres driver 10 | ) 11 | 12 | // Options are the postgres datastore options, reexported here for convenience. 13 | type Options struct { 14 | Host string 15 | Port string 16 | User string 17 | Password string 18 | Database string 19 | Table string 20 | } 21 | 22 | // Queries are the postgres queries for a given table. 23 | type Queries struct { 24 | deleteQuery string 25 | existsQuery string 26 | getQuery string 27 | putQuery string 28 | queryQuery string 29 | prefixQuery string 30 | limitQuery string 31 | offsetQuery string 32 | getSizeQuery string 33 | } 34 | 35 | // NewQueries creates a new PostgreSQL set of queries for the passed table 36 | func NewQueries(tbl string) Queries { 37 | return Queries{ 38 | deleteQuery: fmt.Sprintf("DELETE FROM %s WHERE key = $1", tbl), 39 | existsQuery: fmt.Sprintf("SELECT exists(SELECT 1 FROM %s WHERE key=$1)", tbl), 40 | getQuery: fmt.Sprintf("SELECT data FROM %s WHERE key = $1", tbl), 41 | putQuery: fmt.Sprintf("INSERT INTO %s (key, data) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET data = $2", tbl), 42 | queryQuery: fmt.Sprintf("SELECT key, data FROM %s", tbl), 43 | prefixQuery: ` WHERE key LIKE '%s%%' ORDER BY key`, 44 | limitQuery: ` LIMIT %d`, 45 | offsetQuery: ` OFFSET %d`, 46 | getSizeQuery: fmt.Sprintf("SELECT octet_length(data) FROM %s WHERE key = $1", tbl), 47 | } 48 | } 49 | 50 | // Delete returns the postgres query for deleting a row. 51 | func (q Queries) Delete() string { 52 | return q.deleteQuery 53 | } 54 | 55 | // Exists returns the postgres query for determining if a row exists. 56 | func (q Queries) Exists() string { 57 | return q.existsQuery 58 | } 59 | 60 | // Get returns the postgres query for getting a row. 61 | func (q Queries) Get() string { 62 | return q.getQuery 63 | } 64 | 65 | // Put returns the postgres query for putting a row. 66 | func (q Queries) Put() string { 67 | return q.putQuery 68 | } 69 | 70 | // Query returns the postgres query for getting multiple rows. 71 | func (q Queries) Query() string { 72 | return q.queryQuery 73 | } 74 | 75 | // Prefix returns the postgres query fragment for getting a rows with a key prefix. 76 | func (q Queries) Prefix() string { 77 | return q.prefixQuery 78 | } 79 | 80 | // Limit returns the postgres query fragment for limiting results. 81 | func (q Queries) Limit() string { 82 | return q.limitQuery 83 | } 84 | 85 | // Offset returns the postgres query fragment for returning rows from a given offset. 86 | func (q Queries) Offset() string { 87 | return q.offsetQuery 88 | } 89 | 90 | // GetSize returns the postgres query for determining the size of a value. 91 | func (q Queries) GetSize() string { 92 | return q.getSizeQuery 93 | } 94 | 95 | // Create returns a datastore connected to postgres 96 | func (opts *Options) Create() (*sqlds.Datastore, error) { 97 | opts.setDefaults() 98 | fmtstr := "postgresql:///%s?host=%s&port=%s&user=%s&password=%s&sslmode=disable" 99 | constr := fmt.Sprintf(fmtstr, opts.Database, opts.Host, opts.Port, opts.User, opts.Password) 100 | db, err := sql.Open("postgres", constr) 101 | if err != nil { 102 | return nil, err 103 | } 104 | 105 | return sqlds.NewDatastore(db, NewQueries(opts.Table)), nil 106 | } 107 | 108 | func (opts *Options) setDefaults() { 109 | if opts.Host == "" { 110 | opts.Host = "127.0.0.1" 111 | } 112 | 113 | if opts.Port == "" { 114 | opts.Port = "5432" 115 | } 116 | 117 | if opts.User == "" { 118 | opts.User = "postgres" 119 | } 120 | 121 | if opts.Database == "" { 122 | opts.Database = "datastore" 123 | } 124 | 125 | if opts.Table == "" { 126 | opts.Table = "blocks" 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /sqlite/ds_test.go: -------------------------------------------------------------------------------- 1 | //go:build cgo 2 | 3 | package sqlite 4 | 5 | import ( 6 | "bytes" 7 | "context" 8 | "crypto/rand" 9 | "fmt" 10 | "sort" 11 | "strings" 12 | "testing" 13 | 14 | ds "github.com/ipfs/go-datastore" 15 | dsq "github.com/ipfs/go-datastore/query" 16 | dstest "github.com/ipfs/go-datastore/test" 17 | sqlds "github.com/ipfs/go-ds-sql" 18 | 19 | _ "github.com/mattn/go-sqlite3" 20 | ) 21 | 22 | var testcases = map[string]string{ 23 | "/a": "a", 24 | "/a/b": "ab", 25 | "/a/b/c": "abc", 26 | "/a/b/d": "a/b/d", 27 | "/a/c": "ac", 28 | "/a/d": "ad", 29 | "/e": "e", 30 | "/f": "f", 31 | "/g": "", 32 | } 33 | 34 | // returns datastore, and a function to call on exit. 35 | // 36 | // d, close := newDS(t) 37 | // defer close() 38 | func newDS(t *testing.T) (*sqlds.Datastore, func()) { 39 | t.Helper() 40 | 41 | ds, err := (&Options{}).Create() 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | 46 | return ds, func() { 47 | if err := ds.Close(); err != nil { 48 | t.Fatal(err) 49 | } 50 | } 51 | } 52 | 53 | func addTestCases(t *testing.T, d *sqlds.Datastore, testcases map[string]string) { 54 | t.Helper() 55 | 56 | ctx, cancelCtx := context.WithCancel(context.Background()) 57 | defer cancelCtx() 58 | 59 | for k, v := range testcases { 60 | dsk := ds.NewKey(k) 61 | if err := d.Put(ctx, dsk, []byte(v)); err != nil { 62 | t.Fatal(err) 63 | } 64 | } 65 | 66 | for k, v := range testcases { 67 | dsk := ds.NewKey(k) 68 | v2, err := d.Get(ctx, dsk) 69 | if err != nil { 70 | t.Fatal(err) 71 | } 72 | v2b := v2 73 | if string(v2b) != v { 74 | t.Errorf("%s values differ: %s != %s", k, v, v2) 75 | } 76 | } 77 | } 78 | 79 | func TestQuery(t *testing.T) { 80 | d, done := newDS(t) 81 | defer done() 82 | 83 | ctx, cancelCtx := context.WithCancel(context.Background()) 84 | defer cancelCtx() 85 | 86 | addTestCases(t, d, testcases) 87 | 88 | // test prefix 89 | rs, err := d.Query(ctx, dsq.Query{Prefix: "/a/"}) 90 | if err != nil { 91 | t.Fatal(err) 92 | } 93 | expectMatches(t, []string{ 94 | "/a/b", 95 | "/a/b/c", 96 | "/a/b/d", 97 | "/a/c", 98 | "/a/d", 99 | }, rs) 100 | 101 | // test offset and limit 102 | rs, err = d.Query(ctx, dsq.Query{Prefix: "/a/", Offset: 2, Limit: 2}) 103 | if err != nil { 104 | t.Fatal(err) 105 | } 106 | expectMatches(t, []string{ 107 | "/a/b/d", 108 | "/a/c", 109 | }, rs) 110 | 111 | // test orders 112 | orbk := dsq.OrderByKey{} 113 | orderByKey := []dsq.Order{orbk} 114 | rs, err = d.Query(ctx, dsq.Query{Prefix: "/a/", Orders: orderByKey}) 115 | if err != nil { 116 | t.Fatal(err) 117 | } 118 | expectKeyOrderMatches(t, rs, []string{ 119 | "/a/b", 120 | "/a/b/c", 121 | "/a/b/d", 122 | "/a/c", 123 | "/a/d", 124 | }) 125 | 126 | orbkd := dsq.OrderByKeyDescending{} 127 | orderByDesc := []dsq.Order{orbkd} 128 | rs, err = d.Query(ctx, dsq.Query{Prefix: "/a/", Orders: orderByDesc}) 129 | if err != nil { 130 | t.Fatal(err) 131 | } 132 | expectKeyOrderMatches(t, rs, []string{ 133 | "/a/d", 134 | "/a/c", 135 | "/a/b/d", 136 | "/a/b/c", 137 | "/a/b", 138 | }) 139 | 140 | // test filters 141 | equalFilter := dsq.FilterKeyCompare{Op: dsq.Equal, Key: "/a/b"} 142 | equalFilters := []dsq.Filter{equalFilter} 143 | rs, err = d.Query(ctx, dsq.Query{Prefix: "/a/", Filters: equalFilters}) 144 | if err != nil { 145 | t.Fatal(err) 146 | } 147 | expectKeyFilterMatches(t, rs, []string{"/a/b"}) 148 | 149 | greaterThanFilter := dsq.FilterKeyCompare{Op: dsq.GreaterThan, Key: "/a/b"} 150 | greaterThanFilters := []dsq.Filter{greaterThanFilter} 151 | rs, err = d.Query(ctx, dsq.Query{Prefix: "/a/", Filters: greaterThanFilters}) 152 | if err != nil { 153 | t.Fatal(err) 154 | } 155 | expectKeyFilterMatches(t, rs, []string{ 156 | "/a/b/c", 157 | "/a/b/d", 158 | "/a/c", 159 | "/a/d", 160 | }) 161 | 162 | lessThanFilter := dsq.FilterKeyCompare{Op: dsq.LessThanOrEqual, Key: "/a/b/c"} 163 | lessThanFilters := []dsq.Filter{lessThanFilter} 164 | rs, err = d.Query(ctx, dsq.Query{Prefix: "/a/", Filters: lessThanFilters}) 165 | if err != nil { 166 | t.Fatal(err) 167 | } 168 | expectKeyFilterMatches(t, rs, []string{ 169 | "/a/b", 170 | "/a/b/c", 171 | }) 172 | } 173 | 174 | func TestHas(t *testing.T) { 175 | d, done := newDS(t) 176 | defer done() 177 | 178 | ctx, cancelCtx := context.WithCancel(context.Background()) 179 | defer cancelCtx() 180 | 181 | addTestCases(t, d, testcases) 182 | 183 | has, err := d.Has(ctx, ds.NewKey("/a/b/c")) 184 | if err != nil { 185 | t.Error(err) 186 | } 187 | 188 | if !has { 189 | t.Error("Key should be found") 190 | } 191 | 192 | has, err = d.Has(ctx, ds.NewKey("/a/b/c/d")) 193 | if err != nil { 194 | t.Error(err) 195 | } 196 | 197 | if has { 198 | t.Error("Key should not be found") 199 | } 200 | } 201 | 202 | func TestNotExistGet(t *testing.T) { 203 | d, done := newDS(t) 204 | defer done() 205 | 206 | ctx, cancelCtx := context.WithCancel(context.Background()) 207 | defer cancelCtx() 208 | 209 | addTestCases(t, d, testcases) 210 | 211 | has, err := d.Has(ctx, ds.NewKey("/a/b/c/d")) 212 | if err != nil { 213 | t.Error(err) 214 | } 215 | 216 | if has { 217 | t.Error("Key should not be found") 218 | } 219 | 220 | val, err := d.Get(ctx, ds.NewKey("/a/b/c/d")) 221 | if val != nil { 222 | t.Error("Key should not be found") 223 | } 224 | 225 | if err != ds.ErrNotFound { 226 | t.Error("Error was not set to ds.ErrNotFound") 227 | if err != nil { 228 | t.Error(err) 229 | } 230 | } 231 | } 232 | 233 | func TestDelete(t *testing.T) { 234 | d, done := newDS(t) 235 | defer done() 236 | 237 | ctx, cancelCtx := context.WithCancel(context.Background()) 238 | defer cancelCtx() 239 | 240 | addTestCases(t, d, testcases) 241 | 242 | has, err := d.Has(ctx, ds.NewKey("/a/b/c")) 243 | if err != nil { 244 | t.Error(err) 245 | } 246 | if !has { 247 | t.Error("Key should be found") 248 | } 249 | 250 | err = d.Delete(ctx, ds.NewKey("/a/b/c")) 251 | if err != nil { 252 | t.Error(err) 253 | } 254 | 255 | has, err = d.Has(ctx, ds.NewKey("/a/b/c")) 256 | if err != nil { 257 | t.Error(err) 258 | } 259 | if has { 260 | t.Error("Key should not be found") 261 | } 262 | } 263 | 264 | func TestGetEmpty(t *testing.T) { 265 | d, done := newDS(t) 266 | defer done() 267 | 268 | ctx, cancelCtx := context.WithCancel(context.Background()) 269 | defer cancelCtx() 270 | 271 | err := d.Put(ctx, ds.NewKey("/a"), []byte{}) 272 | if err != nil { 273 | t.Error(err) 274 | } 275 | 276 | v, err := d.Get(ctx, ds.NewKey("/a")) 277 | if err != nil { 278 | t.Error(err) 279 | } 280 | 281 | if len(v) != 0 { 282 | t.Error("expected 0 len []byte form get") 283 | } 284 | } 285 | 286 | func TestBatching(t *testing.T) { 287 | d, done := newDS(t) 288 | defer done() 289 | 290 | ctx, cancelCtx := context.WithCancel(context.Background()) 291 | defer cancelCtx() 292 | 293 | b, err := d.Batch(ctx) 294 | if err != nil { 295 | t.Fatal(err) 296 | } 297 | 298 | for k, v := range testcases { 299 | err := b.Put(ctx, ds.NewKey(k), []byte(v)) 300 | if err != nil { 301 | t.Fatal(err) 302 | } 303 | } 304 | 305 | err = b.Commit(ctx) 306 | if err != nil { 307 | t.Fatal(err) 308 | } 309 | 310 | for k, v := range testcases { 311 | val, err := d.Get(ctx, ds.NewKey(k)) 312 | if err != nil { 313 | t.Fatal(err) 314 | } 315 | 316 | if v != string(val) { 317 | t.Fatal("got wrong data!") 318 | } 319 | } 320 | 321 | //Test delete 322 | b, err = d.Batch(ctx) 323 | if err != nil { 324 | t.Fatal(err) 325 | } 326 | 327 | err = b.Delete(ctx, ds.NewKey("/a/b")) 328 | if err != nil { 329 | t.Fatal(err) 330 | } 331 | 332 | err = b.Delete(ctx, ds.NewKey("/a/b/c")) 333 | if err != nil { 334 | t.Fatal(err) 335 | } 336 | 337 | err = b.Commit(ctx) 338 | if err != nil { 339 | t.Fatal(err) 340 | } 341 | 342 | rs, err := d.Query(ctx, dsq.Query{Prefix: "/"}) 343 | if err != nil { 344 | t.Fatal(err) 345 | } 346 | 347 | expectMatches(t, []string{ 348 | "/a", 349 | "/a/b/d", 350 | "/a/c", 351 | "/a/d", 352 | "/e", 353 | "/f", 354 | "/g", 355 | }, rs) 356 | } 357 | 358 | func SubtestBasicPutGet(t *testing.T) { 359 | d, done := newDS(t) 360 | defer done() 361 | 362 | k := ds.NewKey("foo") 363 | val := []byte("Hello Datastore!") 364 | 365 | ctx, cancelCtx := context.WithCancel(context.Background()) 366 | defer cancelCtx() 367 | 368 | err := d.Put(ctx, k, val) 369 | if err != nil { 370 | t.Fatal("error putting to datastore: ", err) 371 | } 372 | 373 | have, err := d.Has(ctx, k) 374 | if err != nil { 375 | t.Fatal("error calling has on key we just put: ", err) 376 | } 377 | 378 | if !have { 379 | t.Fatal("should have key foo, has returned false") 380 | } 381 | 382 | size, err := d.GetSize(ctx, k) 383 | if err != nil { 384 | t.Fatal("error getting size after put: ", err) 385 | } 386 | if size != len(val) { 387 | t.Fatalf("incorrect size: expected %d, got %d", len(val), size) 388 | } 389 | 390 | out, err := d.Get(ctx, k) 391 | if err != nil { 392 | t.Fatal("error getting value after put: ", err) 393 | } 394 | 395 | if !bytes.Equal(out, val) { 396 | t.Fatal("value received on get wasnt what we expected:", out) 397 | } 398 | 399 | have, err = d.Has(ctx, k) 400 | if err != nil { 401 | t.Fatal("error calling has after get: ", err) 402 | } 403 | 404 | if !have { 405 | t.Fatal("should have key foo, has returned false") 406 | } 407 | 408 | size, err = d.GetSize(ctx, k) 409 | if err != nil { 410 | t.Fatal("error getting size after get: ", err) 411 | } 412 | if size != len(val) { 413 | t.Fatalf("incorrect size: expected %d, got %d", len(val), size) 414 | } 415 | 416 | err = d.Delete(ctx, k) 417 | if err != nil { 418 | t.Fatal("error calling delete: ", err) 419 | } 420 | 421 | have, err = d.Has(ctx, k) 422 | if err != nil { 423 | t.Fatal("error calling has after delete: ", err) 424 | } 425 | 426 | if have { 427 | t.Fatal("should not have key foo, has returned true") 428 | } 429 | 430 | size, err = d.GetSize(ctx, k) 431 | switch err { 432 | case ds.ErrNotFound: 433 | case nil: 434 | t.Fatal("expected error getting size after delete") 435 | default: 436 | t.Fatal("wrong error getting size after delete: ", err) 437 | } 438 | if size != -1 { 439 | t.Fatal("expected missing size to be -1") 440 | } 441 | } 442 | 443 | func TestNotFounds(t *testing.T) { 444 | d, done := newDS(t) 445 | defer done() 446 | 447 | badk := ds.NewKey("notreal") 448 | 449 | ctx, cancelCtx := context.WithCancel(context.Background()) 450 | defer cancelCtx() 451 | 452 | val, err := d.Get(ctx, badk) 453 | if err != ds.ErrNotFound { 454 | t.Fatal("expected ErrNotFound for key that doesnt exist, got: ", err) 455 | } 456 | 457 | if val != nil { 458 | t.Fatal("get should always return nil for not found values") 459 | } 460 | 461 | have, err := d.Has(ctx, badk) 462 | if err != nil { 463 | t.Fatal("error calling has on not found key: ", err) 464 | } 465 | if have { 466 | t.Fatal("has returned true for key we don't have") 467 | } 468 | 469 | size, err := d.GetSize(ctx, badk) 470 | switch err { 471 | case ds.ErrNotFound: 472 | case nil: 473 | t.Fatal("expected error getting size after delete") 474 | default: 475 | t.Fatal("wrong error getting size after delete: ", err) 476 | } 477 | if size != -1 { 478 | t.Fatal("expected missing size to be -1") 479 | } 480 | } 481 | 482 | func SubtestManyKeysAndQuery(t *testing.T) { 483 | d, done := newDS(t) 484 | defer done() 485 | 486 | ctx, cancelCtx := context.WithCancel(context.Background()) 487 | defer cancelCtx() 488 | 489 | var keys []ds.Key 490 | var keystrs []string 491 | var values [][]byte 492 | count := 100 493 | for i := 0; i < count; i++ { 494 | s := fmt.Sprintf("%dkey%d", i, i) 495 | dsk := ds.NewKey(s) 496 | keystrs = append(keystrs, dsk.String()) 497 | keys = append(keys, dsk) 498 | buf := make([]byte, 64) 499 | _, _ = rand.Read(buf) 500 | values = append(values, buf) 501 | } 502 | 503 | t.Logf("putting %d values", count) 504 | for i, k := range keys { 505 | err := d.Put(ctx, k, values[i]) 506 | if err != nil { 507 | t.Fatalf("error on put[%d]: %s", i, err) 508 | } 509 | } 510 | 511 | t.Log("getting values back") 512 | for i, k := range keys { 513 | val, err := d.Get(ctx, k) 514 | if err != nil { 515 | t.Fatalf("error on get[%d]: %s", i, err) 516 | } 517 | 518 | if !bytes.Equal(val, values[i]) { 519 | t.Fatal("input value didnt match the one returned from Get") 520 | } 521 | } 522 | 523 | t.Log("querying values") 524 | q := dsq.Query{KeysOnly: true} 525 | resp, err := d.Query(ctx, q) 526 | if err != nil { 527 | t.Fatal("calling query: ", err) 528 | } 529 | 530 | t.Log("aggregating query results") 531 | var outkeys []string 532 | for { 533 | res, ok := resp.NextSync() 534 | if res.Error != nil { 535 | t.Fatal("query result error: ", res.Error) 536 | } 537 | if !ok { 538 | break 539 | } 540 | 541 | outkeys = append(outkeys, res.Key) 542 | } 543 | 544 | t.Log("verifying query output") 545 | sort.Strings(keystrs) 546 | sort.Strings(outkeys) 547 | 548 | if len(keystrs) != len(outkeys) { 549 | t.Fatal("got wrong number of keys back") 550 | } 551 | 552 | for i, s := range keystrs { 553 | if outkeys[i] != s { 554 | t.Fatalf("in key output, got %s but expected %s", outkeys[i], s) 555 | } 556 | } 557 | 558 | t.Log("deleting all keys") 559 | for _, k := range keys { 560 | if err := d.Delete(ctx, k); err != nil { 561 | t.Fatal(err) 562 | } 563 | } 564 | } 565 | 566 | // Tests from basic_tests from go-datastore 567 | func TestBasicPutGet(t *testing.T) { 568 | d, done := newDS(t) 569 | defer done() 570 | 571 | ctx, cancelCtx := context.WithCancel(context.Background()) 572 | defer cancelCtx() 573 | 574 | k := ds.NewKey("foo") 575 | val := []byte("Hello Datastore!") 576 | 577 | err := d.Put(ctx, k, val) 578 | if err != nil { 579 | t.Fatal("error putting to datastore: ", err) 580 | } 581 | 582 | have, err := d.Has(ctx, k) 583 | if err != nil { 584 | t.Fatal("error calling has on key we just put: ", err) 585 | } 586 | 587 | if !have { 588 | t.Fatal("should have key foo, has returned false") 589 | } 590 | 591 | out, err := d.Get(ctx, k) 592 | if err != nil { 593 | t.Fatal("error getting value after put: ", err) 594 | } 595 | 596 | if !bytes.Equal(out, val) { 597 | t.Fatal("value received on get wasnt what we expected:", out) 598 | } 599 | 600 | have, err = d.Has(ctx, k) 601 | if err != nil { 602 | t.Fatal("error calling has after get: ", err) 603 | } 604 | 605 | if !have { 606 | t.Fatal("should have key foo, has returned false") 607 | } 608 | 609 | err = d.Delete(ctx, k) 610 | if err != nil { 611 | t.Fatal("error calling delete: ", err) 612 | } 613 | 614 | have, err = d.Has(ctx, k) 615 | if err != nil { 616 | t.Fatal("error calling has after delete: ", err) 617 | } 618 | 619 | if have { 620 | t.Fatal("should not have key foo, has returned true") 621 | } 622 | SubtestBasicPutGet(t) 623 | } 624 | 625 | func TestManyKeysAndQuery(t *testing.T) { 626 | d, done := newDS(t) 627 | defer done() 628 | 629 | ctx, cancelCtx := context.WithCancel(context.Background()) 630 | defer cancelCtx() 631 | 632 | var keys []ds.Key 633 | var keystrs []string 634 | var values [][]byte 635 | count := 100 636 | for i := 0; i < count; i++ { 637 | s := fmt.Sprintf("%dkey%d", i, i) 638 | dsk := ds.NewKey(s) 639 | keystrs = append(keystrs, dsk.String()) 640 | keys = append(keys, dsk) 641 | buf := make([]byte, 64) 642 | _, _ = rand.Read(buf) 643 | values = append(values, buf) 644 | } 645 | 646 | t.Logf("putting %d values", count) 647 | for i, k := range keys { 648 | err := d.Put(ctx, k, values[i]) 649 | if err != nil { 650 | t.Fatalf("error on put[%d]: %s", i, err) 651 | } 652 | } 653 | 654 | t.Log("getting values back") 655 | for i, k := range keys { 656 | val, err := d.Get(ctx, k) 657 | if err != nil { 658 | t.Fatalf("error on get[%d]: %s", i, err) 659 | } 660 | 661 | if !bytes.Equal(val, values[i]) { 662 | t.Fatal("input value didnt match the one returned from Get") 663 | } 664 | } 665 | 666 | t.Log("querying values") 667 | q := dsq.Query{KeysOnly: true} 668 | resp, err := d.Query(ctx, q) 669 | if err != nil { 670 | t.Fatal("calling query: ", err) 671 | } 672 | 673 | t.Log("aggregating query results") 674 | var outkeys []string 675 | for { 676 | res, ok := resp.NextSync() 677 | if res.Error != nil { 678 | t.Fatal("query result error: ", res.Error) 679 | } 680 | if !ok { 681 | break 682 | } 683 | 684 | outkeys = append(outkeys, res.Key) 685 | } 686 | 687 | t.Log("verifying query output") 688 | sort.Strings(keystrs) 689 | sort.Strings(outkeys) 690 | 691 | if len(keystrs) != len(outkeys) { 692 | t.Fatalf("got wrong number of keys back, %d != %d", len(keystrs), len(outkeys)) 693 | } 694 | 695 | for i, s := range keystrs { 696 | if outkeys[i] != s { 697 | t.Fatalf("in key output, got %s but expected %s", outkeys[i], s) 698 | } 699 | } 700 | 701 | t.Log("deleting all keys") 702 | for _, k := range keys { 703 | if err := d.Delete(ctx, k); err != nil { 704 | t.Fatal(err) 705 | } 706 | } 707 | 708 | SubtestManyKeysAndQuery(t) 709 | } 710 | 711 | func TestSuite(t *testing.T) { 712 | d, done := newDS(t) 713 | defer done() 714 | 715 | dstest.SubtestAll(t, d) 716 | } 717 | 718 | func expectMatches(t *testing.T, expect []string, actualR dsq.Results) { 719 | t.Helper() 720 | actual, err := actualR.Rest() 721 | if err != nil { 722 | t.Error(err) 723 | } 724 | 725 | if len(actual) != len(expect) { 726 | t.Error("not enough", expect, actual) 727 | } 728 | for _, k := range expect { 729 | found := false 730 | for _, e := range actual { 731 | if e.Key == k { 732 | found = true 733 | } 734 | } 735 | if !found { 736 | t.Error(k, "not found") 737 | } 738 | } 739 | } 740 | 741 | func expectKeyOrderMatches(t *testing.T, actual dsq.Results, expect []string) { 742 | rs, err := actual.Rest() 743 | if err != nil { 744 | t.Error("error fetching dsq.Results", expect, actual) 745 | return 746 | } 747 | 748 | if len(rs) != len(expect) { 749 | t.Error("expect != actual.", expect, actual) 750 | return 751 | } 752 | 753 | for i, r := range rs { 754 | if r.Key != expect[i] { 755 | t.Error("expect != actual.", expect, actual) 756 | return 757 | } 758 | } 759 | } 760 | 761 | func expectKeyFilterMatches(t *testing.T, actual dsq.Results, expect []string) { 762 | actualE, err := actual.Rest() 763 | if err != nil { 764 | t.Error(err) 765 | return 766 | } 767 | actualS := make([]string, len(actualE)) 768 | for i, e := range actualE { 769 | actualS[i] = e.Key 770 | } 771 | 772 | if len(actualS) != len(expect) { 773 | t.Error("length doesn't match.", expect, actualS) 774 | return 775 | } 776 | 777 | if strings.Join(actualS, "") != strings.Join(expect, "") { 778 | t.Error("expect != actual.", expect, actualS) 779 | return 780 | } 781 | } 782 | -------------------------------------------------------------------------------- /sqlite/sqlite.go: -------------------------------------------------------------------------------- 1 | package sqlite 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/hex" 6 | "fmt" 7 | "strings" 8 | 9 | sqlds "github.com/ipfs/go-ds-sql" 10 | // we don't import a specific driver to let the user choose 11 | ) 12 | 13 | // Options are the sqlite datastore options, reexported here for convenience. 14 | type Options struct { 15 | Driver string 16 | DSN string 17 | Table string 18 | // Don't try to create table 19 | NoCreate bool 20 | 21 | // sqlcipher extension specific 22 | Key []byte 23 | CipherPageSize uint 24 | } 25 | 26 | // Queries are the sqlite queries for a given table. 27 | type Queries struct { 28 | deleteQuery string 29 | existsQuery string 30 | getQuery string 31 | putQuery string 32 | queryQuery string 33 | prefixQuery string 34 | limitQuery string 35 | offsetQuery string 36 | getSizeQuery string 37 | } 38 | 39 | // NewQueries creates a new sqlite set of queries for the passed table 40 | func NewQueries(tbl string) Queries { 41 | return Queries{ 42 | deleteQuery: fmt.Sprintf("DELETE FROM %s WHERE key = $1", tbl), 43 | existsQuery: fmt.Sprintf("SELECT exists(SELECT 1 FROM %s WHERE key=$1)", tbl), 44 | getQuery: fmt.Sprintf("SELECT data FROM %s WHERE key = $1", tbl), 45 | putQuery: fmt.Sprintf("INSERT OR REPLACE INTO %s(key, data) VALUES($1, $2)", tbl), 46 | queryQuery: fmt.Sprintf("SELECT key, data FROM %s", tbl), 47 | prefixQuery: ` WHERE key GLOB '%s*' ORDER BY key`, 48 | limitQuery: ` LIMIT %d`, 49 | offsetQuery: ` OFFSET %d`, 50 | getSizeQuery: fmt.Sprintf("SELECT length(data) FROM %s WHERE key = $1", tbl), 51 | } 52 | } 53 | 54 | // Delete returns the sqlite query for deleting a row. 55 | func (q Queries) Delete() string { 56 | return q.deleteQuery 57 | } 58 | 59 | // Exists returns the sqlite query for determining if a row exists. 60 | func (q Queries) Exists() string { 61 | return q.existsQuery 62 | } 63 | 64 | // Get returns the sqlite query for getting a row. 65 | func (q Queries) Get() string { 66 | return q.getQuery 67 | } 68 | 69 | // Put returns the sqlite query for putting a row. 70 | func (q Queries) Put() string { 71 | return q.putQuery 72 | } 73 | 74 | // Query returns the sqlite query for getting multiple rows. 75 | func (q Queries) Query() string { 76 | return q.queryQuery 77 | } 78 | 79 | // Prefix returns the sqlite query fragment for getting a rows with a key prefix. 80 | func (q Queries) Prefix() string { 81 | return q.prefixQuery 82 | } 83 | 84 | // Limit returns the sqlite query fragment for limiting results. 85 | func (q Queries) Limit() string { 86 | return q.limitQuery 87 | } 88 | 89 | // Offset returns the sqlite query fragment for returning rows from a given offset. 90 | func (q Queries) Offset() string { 91 | return q.offsetQuery 92 | } 93 | 94 | // GetSize returns the sqlite query for determining the size of a value. 95 | func (q Queries) GetSize() string { 96 | return q.getSizeQuery 97 | } 98 | 99 | // Create returns a datastore connected to sqlite 100 | func (opts *Options) Create() (*sqlds.Datastore, error) { 101 | opts.setDefaults() 102 | 103 | args := []string{} 104 | if len(opts.Key) != 0 { 105 | // sqlcipher expects a 32 bytes key 106 | if len(opts.Key) != 32 { 107 | return nil, fmt.Errorf("bad key length, expected 32 bytes, got %d", len(opts.Key)) 108 | } 109 | args = append(args, fmt.Sprintf("_pragma_key=x'%s'", hex.EncodeToString(opts.Key))) 110 | args = append(args, fmt.Sprintf("_pragma_cipher_page_size=%d", opts.CipherPageSize)) 111 | } 112 | dsn := opts.DSN 113 | if len(args) != 0 { 114 | if strings.ContainsRune(dsn, '?') { 115 | dsn += "&" 116 | } else { 117 | dsn += "?" 118 | } 119 | dsn += strings.Join(args, "&") 120 | } 121 | 122 | db, err := sql.Open(opts.Driver, dsn) 123 | if err != nil { 124 | return nil, fmt.Errorf("failed to open database: %w", err) 125 | } 126 | 127 | if err := db.Ping(); err != nil { 128 | _ = db.Close() 129 | return nil, fmt.Errorf("failed to ping database: %w", err) 130 | } 131 | 132 | if !opts.NoCreate { 133 | if _, err := db.Exec(fmt.Sprintf(` 134 | CREATE TABLE IF NOT EXISTS %s ( 135 | key TEXT PRIMARY KEY, 136 | data BLOB 137 | ) WITHOUT ROWID; 138 | `, opts.Table)); err != nil { 139 | _ = db.Close() 140 | return nil, fmt.Errorf("failed to ensure table exists: %w", err) 141 | } 142 | } 143 | 144 | return sqlds.NewDatastore(db, NewQueries(opts.Table)), nil 145 | } 146 | 147 | func (opts *Options) setDefaults() { 148 | if opts.Driver == "" { 149 | opts.Driver = "sqlite3" 150 | } 151 | 152 | if opts.DSN == "" { 153 | opts.DSN = ":memory:" 154 | } 155 | 156 | if len(opts.Key) != 0 && opts.CipherPageSize == 0 { 157 | opts.CipherPageSize = 4096 158 | } 159 | 160 | if opts.Table == "" { 161 | opts.Table = "blocks" 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /txn.go: -------------------------------------------------------------------------------- 1 | package sqlds 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | 8 | datastore "github.com/ipfs/go-datastore" 9 | dsq "github.com/ipfs/go-datastore/query" 10 | ) 11 | 12 | // ErrNotImplemented is returned when the SQL datastore does not yet implement the function call. 13 | var ErrNotImplemented = fmt.Errorf("not implemented") 14 | 15 | type txn struct { 16 | db *sql.DB 17 | queries Queries 18 | txn *sql.Tx 19 | } 20 | 21 | // NewTransaction creates a new database transaction, note the readOnly parameter is ignored by this implementation. 22 | func (ds *Datastore) NewTransaction(ctx context.Context, _ bool) (datastore.Txn, error) { 23 | sqlTxn, err := ds.db.BeginTx(ctx, nil) 24 | if err != nil { 25 | if sqlTxn != nil { 26 | // nothing we can do about this error. 27 | _ = sqlTxn.Rollback() 28 | } 29 | 30 | return nil, err 31 | } 32 | 33 | return &txn{ 34 | db: ds.db, 35 | queries: ds.queries, 36 | txn: sqlTxn, 37 | }, nil 38 | } 39 | 40 | func (t *txn) Get(ctx context.Context, key datastore.Key) ([]byte, error) { 41 | row := t.txn.QueryRowContext(ctx, t.queries.Get(), key.String()) 42 | var out []byte 43 | 44 | switch err := row.Scan(&out); err { 45 | case sql.ErrNoRows: 46 | return nil, datastore.ErrNotFound 47 | case nil: 48 | return out, nil 49 | default: 50 | return nil, err 51 | } 52 | } 53 | 54 | func (t *txn) Has(ctx context.Context, key datastore.Key) (bool, error) { 55 | row := t.txn.QueryRowContext(ctx, t.queries.Exists(), key.String()) 56 | var exists bool 57 | 58 | switch err := row.Scan(&exists); err { 59 | case sql.ErrNoRows: 60 | return exists, nil 61 | case nil: 62 | return exists, nil 63 | default: 64 | return exists, err 65 | } 66 | } 67 | 68 | func (t *txn) GetSize(ctx context.Context, key datastore.Key) (int, error) { 69 | row := t.txn.QueryRowContext(ctx, t.queries.GetSize(), key.String()) 70 | var size int 71 | 72 | switch err := row.Scan(&size); err { 73 | case sql.ErrNoRows: 74 | return -1, datastore.ErrNotFound 75 | case nil: 76 | return size, nil 77 | default: 78 | return 0, err 79 | } 80 | } 81 | 82 | func (t *txn) Query(ctx context.Context, q dsq.Query) (dsq.Results, error) { 83 | return nil, ErrNotImplemented 84 | } 85 | 86 | // Put adds a value to the datastore identified by the given key. 87 | func (t *txn) Put(ctx context.Context, key datastore.Key, val []byte) error { 88 | _, err := t.txn.ExecContext(ctx, t.queries.Put(), key.String(), val) 89 | if err != nil { 90 | _ = t.txn.Rollback() 91 | return err 92 | } 93 | return nil 94 | } 95 | 96 | // Delete removes a value from the datastore that matches the given key. 97 | func (t *txn) Delete(ctx context.Context, key datastore.Key) error { 98 | _, err := t.txn.ExecContext(ctx, t.queries.Delete(), key.String()) 99 | if err != nil { 100 | _ = t.txn.Rollback() 101 | return err 102 | } 103 | return nil 104 | } 105 | 106 | // Commit finalizes a transaction. 107 | func (t *txn) Commit(ctx context.Context) error { 108 | err := t.txn.Commit() 109 | if err != nil { 110 | _ = t.txn.Rollback() 111 | return err 112 | } 113 | return nil 114 | } 115 | 116 | // Discard throws away changes recorded in a transaction without committing 117 | // them to the underlying Datastore. 118 | func (t *txn) Discard(ctx context.Context) { 119 | _ = t.txn.Rollback() 120 | } 121 | 122 | var _ datastore.TxnDatastore = (*Datastore)(nil) 123 | -------------------------------------------------------------------------------- /version.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "v0.3.2" 3 | } 4 | --------------------------------------------------------------------------------