├── .github └── workflows │ └── tests.yml ├── .gitignore ├── LICENSE ├── README.md ├── base ├── base.go ├── errors.go └── registry.go ├── db-hierarchy.gv ├── db-hierarchy.svg ├── doc.go ├── docs ├── README.md ├── kv-flat.md ├── kv-hierarchical.md ├── sql-tuple.md └── tuple-strict.md ├── filter └── filters.go ├── go.mod ├── go.sum ├── kv ├── all │ └── all.go ├── bbolt │ ├── bolt.go │ └── bolt_test.go ├── bolt │ ├── bolt.go │ └── bolt_test.go ├── flat │ ├── all │ │ ├── all.go │ │ └── all_64.go │ ├── badger │ │ ├── badger.go │ │ └── badger_test.go │ ├── btree │ │ ├── Makefile │ │ ├── btree.go │ │ ├── btree_test.go │ │ └── keys.go │ ├── helpers.go │ ├── iterator.go │ ├── kv.go │ ├── leveldb │ │ ├── leveldb.go │ │ └── leveldb_test.go │ ├── pebble │ │ ├── pebble.go │ │ └── pebble_test.go │ ├── registry.go │ ├── upgrade.go │ └── upgrade_test.go ├── helpers.go ├── iterator.go ├── kv.go ├── kv_test.go ├── kvdebug │ └── kvdebug.go ├── kvtest │ ├── helpers.go │ └── kvtest.go ├── options │ ├── options.go │ ├── prefix_flat.go │ └── prefix_kv.go └── registry.go ├── legacy └── nosql │ ├── all │ └── all.go │ ├── couch │ ├── couch.go │ ├── couch_test.go │ ├── ouch.go │ ├── ouch_test.go │ ├── pouch.go │ ├── pouch_test.go │ ├── test │ │ ├── couchtest.go │ │ └── pouchtest.go │ └── values.go │ ├── elastic │ ├── elastic.go │ ├── elastic_test.go │ └── test │ │ └── elastictest.go │ ├── mongo │ ├── mongo.go │ ├── mongo_test.go │ └── test │ │ └── mongotest.go │ ├── nosql.go │ ├── nosqltest │ ├── all │ │ └── alltest.go │ ├── nosqltest.go │ └── registry.go │ ├── registry.go │ ├── value.go │ └── value_test.go ├── tuple ├── datastore │ ├── datastore.go │ └── datastore_test.go ├── helpers.go ├── kv │ ├── downgrade.go │ ├── upgrade.go │ └── upgrade_test.go ├── sql │ ├── all │ │ └── all.go │ ├── builder.go │ ├── dialect.go │ ├── mysql │ │ ├── mysql.go │ │ ├── mysql_test.go │ │ └── test │ │ │ └── mysqltest.go │ ├── postgres │ │ ├── postgres.go │ │ ├── postgres_test.go │ │ └── test │ │ │ └── postgrestest.go │ ├── registry.go │ ├── sql.go │ └── sqltest │ │ ├── all │ │ └── alltest.go │ │ ├── registry.go │ │ └── sqltest.go ├── stats.go ├── tuple.go ├── tuplepb │ ├── tuple.pb.go │ ├── tuple.proto │ ├── tuplepb.go │ └── tuplepb_test.go └── tupletest │ ├── helpers.go │ └── tupletest.go └── values ├── types.go ├── values.go └── values_test.go /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v3 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v4 18 | with: 19 | go-version: '1.19' 20 | 21 | - name: Vet 22 | run: go vet ./... 23 | 24 | - name: Test 25 | run: go test -v ./... 26 | 27 | build-32: 28 | runs-on: ubuntu-latest 29 | steps: 30 | - uses: actions/checkout@v3 31 | 32 | - name: Set up Go 33 | uses: actions/setup-go@v4 34 | with: 35 | go-version: '1.19' 36 | 37 | - name: Vet 38 | env: 39 | GOARCH: 386 40 | run: go vet ./... 41 | 42 | - name: Test 43 | env: 44 | GOARCH: 386 45 | run: go test -v ./... 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### Go template 3 | # Binaries for programs and plugins 4 | *.exe 5 | *.exe~ 6 | *.dll 7 | *.so 8 | *.dylib 9 | 10 | # Test binary, build with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | vendor/ 17 | 18 | ### JetBrains template 19 | 20 | # User-specific stuff 21 | .idea/ 22 | .vscode/ 23 | 24 | # File-based project format 25 | *.iws 26 | 27 | # IntelliJ 28 | out/ 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HiDAL-Go 2 | 3 | [![Go Reference](https://pkg.go.dev/badge/github.com/hidal-go/hidalgo.svg)](https://pkg.go.dev/github.com/hidal-go/hidalgo) 4 | 5 | HiDAL-Go = **Hi**gh-level **D**atabase **A**bstraction **L**ayer for **Go** 6 | 7 | This library consists of multiple abstraction layers over existing databases, 8 | either embedded or not. 9 | 10 | Diagram of available implementations: 11 | 12 | ![DB hierarchy](db-hierarchy.svg) 13 | 14 | The unique feature of this library is that backends with lower abstraction levels 15 | (e.g. key-value stores) can implement higher abstraction levels (e.g. tuple store). 16 | 17 | It is also possible to go in the other direction: given a tuple store (e.g. SQL), 18 | it is possible to "downgrade" it to a KV store. The KV abstraction will be the same 19 | as if a regular KV store is used. 20 | 21 | See [docs](docs/README.md) for more details on available implementations. 22 | -------------------------------------------------------------------------------- /base/base.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | import "context" 4 | 5 | // DB is a common interface implemented by all database abstractions. 6 | type DB interface { 7 | // Close closes the database. 8 | Close() error 9 | } 10 | 11 | // Tx is a common interface implemented by all transactions. 12 | type Tx interface { 13 | // Commit applies all changes made in the transaction. 14 | Commit(ctx context.Context) error 15 | // Close rolls back the transaction. 16 | // Committed transactions will not be affected by calling Close. 17 | Close() error 18 | } 19 | 20 | // Iterator is a common interface implemented by all iterators. 21 | type Iterator interface { 22 | // Next advances an iterator. 23 | Next(ctx context.Context) bool 24 | // Err returns a last encountered error. 25 | Err() error 26 | // Close frees resources. 27 | Close() error 28 | } 29 | -------------------------------------------------------------------------------- /base/errors.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | import "fmt" 4 | 5 | // ErrVolatile is returned when trying to pass a path for opening an in-memory database. 6 | var ErrVolatile = fmt.Errorf("database is in-memory") 7 | 8 | var _ error = ErrRegistered{} 9 | 10 | // ErrRegistered is thrown when trying to register a database driver with a name that is already registered. 11 | type ErrRegistered struct { 12 | Name string 13 | } 14 | 15 | func (e ErrRegistered) Error() string { 16 | return "already registered: " + e.Name 17 | } 18 | -------------------------------------------------------------------------------- /base/registry.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | // RegistrySep is a name separator for building implementation hierarchy. 4 | const RegistrySep = "." 5 | 6 | // Registration is a common information about the database driver. 7 | type Registration struct { 8 | Name string // unique name 9 | Title string // human-readable name 10 | Local bool // stores data on local disk or keeps it in-memory 11 | Volatile bool // not persistent 12 | } 13 | -------------------------------------------------------------------------------- /db-hierarchy.gv: -------------------------------------------------------------------------------- 1 | # Generate SVG using: 2 | # dot db-hierarchy.gv -T svg > db-hierarchy.svg 3 | 4 | digraph DBs { 5 | rankdir="BT" 6 | 7 | flat_kv [label="Flat KV" URL="./docs/kv-flat.md" color="#bbbbff" style=filled] 8 | flat_kv -> hie_kv 9 | badger [label="Badger"] 10 | badger -> flat_kv 11 | btree [label="B-Tree"] 12 | btree -> flat_kv 13 | leveldb [label="LevelDB"] 14 | leveldb -> flat_kv 15 | pebble [label="Pebble"] 16 | pebble -> flat_kv 17 | 18 | hie_kv [label="Hierarchical KV" URL="./docs/kv-hierarchical.md" color="#bbbbff" style=filled] 19 | hie_kv -> strict_tuple 20 | hie_kv -> flat_kv [style=dashed] 21 | bolt [label="Bolt"] 22 | bolt -> hie_kv 23 | bbolt [label="BBolt"] 24 | bbolt -> hie_kv 25 | 26 | sql_tuple [label="SQL" URL="./docs/sql-tuple.md"] 27 | sql_tuple -> strict_tuple 28 | mysql [label="MySQL"] 29 | mysql -> sql_tuple 30 | postgres [label="PostgreSQL"] 31 | postgres -> sql_tuple 32 | 33 | datastore [label="Google\nDatastore"] 34 | datastore -> strict_tuple 35 | 36 | strict_tuple [label="Tuple store" URL="./docs/tuple-strict.md" color="#bbbbff" style=filled] 37 | strict_tuple -> flat_kv [style=dashed] 38 | 39 | legacy_nosql [label="NoSQL\n(legacy)", color="#dddddd" style=filled] 40 | legacy_mongo [label="MongoDB", color=grey] 41 | legacy_mongo -> legacy_nosql [color=grey] 42 | legacy_elastic [label="ElasticSearch", color=grey] 43 | legacy_elastic -> legacy_nosql [color=grey] 44 | legacy_ouch [label="go-kivik", color=grey] 45 | legacy_ouch -> legacy_nosql [color=grey] 46 | legacy_couch [label="CouchDB", color=grey] 47 | legacy_couch -> legacy_ouch [color=grey] 48 | legacy_pouch [label="PouchDB", color=grey] 49 | legacy_pouch -> legacy_ouch [color=grey] 50 | } -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package hidalgo provides high-level database abstractions over existing databases. 2 | // 3 | // See subpackages for more information. 4 | package hidalgo 5 | 6 | //go:generate dot -Tsvg -odb-hierarchy.svg db-hierarchy.gv 7 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Documentation 2 | 3 | For Go documentation, see 4 | [![Go Reference](https://pkg.go.dev/badge/github.com/hidal-go/hidalgo.svg "GoDoc")](https://pkg.go.dev/github.com/hidal-go/hidalgo) 5 | 6 | ## Abstractions 7 | 8 | * [Flat KV](kv-flat.md) 9 | * [Hierarchical KV](kv-hierarchical.md) 10 | * [Tuple store](tuple-strict.md) 11 | -------------------------------------------------------------------------------- /docs/kv-flat.md: -------------------------------------------------------------------------------- 1 | # Flat key-value store 2 | 3 | [![Go Reference](https://pkg.go.dev/badge/github.com/hidal-go/hidalgo/kv/flat.svg "GoDoc for flat key-value store within HiDAL-Go")](https://pkg.go.dev/github.com/hidal-go/hidalgo/kv/flat) 4 | 5 | Flat KV store is the most basic abstraction - a store that associates a single binary key with a single binary value. 6 | 7 | By using a specific key separator these stores can implement [hierarchical key-value store](kv-hierarchical.md). 8 | 9 | ## Supported backends 10 | 11 | * In-memory [B-Tree](https://github.com/cznic/b) 12 | * [Badger](https://github.com/dgraph-io/badger) 13 | * [Pebble](https://github.com/cockroachdb/pebble) (experimental) 14 | * [LevelDB](https://github.com/syndtr/goleveldb) 15 | 16 | * Downgrade of [Hierarchical KV](kv-hierarchical.md) 17 | * Downgrade of [Tuple store](tuple-strict.md) 18 | 19 | ## Backend features 20 | 21 | | Backend | Persistence | Concurrency | Transactions | 22 | |-------------------------------|-------------|-------------|--------------| 23 | | B-Tree | - | - | - | 24 | | Badger | X | X | X | 25 | | Pebble | X | X | - | 26 | | LevelDB | X | X | X | 27 | | [Hie. KV](kv-hierarchical.md) | X | X | X | 28 | | [Tuple](tuple-strict.md) | X | X | - | 29 | 30 | ## Backend optimizations 31 | 32 | | Backend | Seek | Prefix | 33 | |-------------------------------|------|--------| 34 | | B-Tree | X | X | 35 | | Badger | X | X | 36 | | Pebble | X | X | 37 | | LevelDB | X | X | 38 | | [Hie. KV](kv-hierarchical.md) | X | X | 39 | | [Tuple](tuple-strict.md) | X | X | 40 | 41 | ## Notes 42 | 43 | * Even though all backends expose `Tx` interface, some may behave incorrectly 44 | during concurrent writes to the same key. This is why transactions support 45 | may be marked unavailable for some backends. Contributions welcome :) 46 | * Support for any of the features in meta-backends like tuple store will depend 47 | on when the underlying backend support. 48 | * Some features may be marked as not implemented for meta backend in this table, 49 | which means that they will not yet work for any of the underlying backends. 50 | -------------------------------------------------------------------------------- /docs/kv-hierarchical.md: -------------------------------------------------------------------------------- 1 | # Hierarchical key-value store 2 | 3 | [![Go Reference](https://pkg.go.dev/badge/github.com/hidal-go/hidalgo/kv.svg "GoDoc for hierarchical key-value store within HiDAL-Go")](https://pkg.go.dev/github.com/hidal-go/hidalgo/kv) 4 | 5 | One of the most basic abstractions for a database index - a store that associates a composite 6 | binary key with a single binary value. 7 | 8 | This abstraction assumes that store can isolate key-value space into named hierarchy of "buckets", 9 | hence the key is composed of names of these buckets (a path). 10 | 11 | By storing a schema and using row serialization, these stores can implement [tuple store](tuple-strict.md). 12 | 13 | ## Supported backends 14 | 15 | * [Bolt](https://github.com/boltdb/bolt) 16 | * [BBolt](https://github.com/coreos/bbolt) 17 | * Emulated over [Flat KV](kv-flat.md) 18 | 19 | ## Backend features 20 | 21 | | Backend | Persistence | Concurrency | Transactions | 22 | |-----------------------|-------------|-------------|--------------| 23 | | Bolt | X | X | X | 24 | | BBolt | X | X | X | 25 | | [Flat KV](kv-flat.md) | X | X | X | 26 | 27 | ## Backend optimizations 28 | 29 | | Backend | Seek | Prefix | 30 | |-----------------------|------|--------| 31 | | Bolt | X | X | 32 | | BBolt | X | X | 33 | | [Flat KV](kv-flat.md) | X | X | 34 | 35 | ## Notes 36 | 37 | * Even though all backends expose `Tx` interface, some may behave incorrectly 38 | during concurrent writes to the same key. This is why transactions support 39 | may be marked unavailable for some backends. Contributions welcome :) 40 | * Support for any of the features in meta-backends like flat KV store will depend 41 | on when the underlying backend support. 42 | * Some features may be marked as not implemented for meta backend in this table, 43 | which means that they will not yet work for any of the underlying backends. 44 | -------------------------------------------------------------------------------- /docs/sql-tuple.md: -------------------------------------------------------------------------------- 1 | # SQL tuple store 2 | 3 | [![Go Reference](https://pkg.go.dev/badge/github.com/hidal-go/hidalgo/tuple/sql.svg "GoDoc for SQL tuple store within HiDAL-Go")](https://pkg.go.dev/github.com/hidal-go/hidalgo/tuple/sql) 4 | 5 | SQL provides a natural way to store tuples indexed by a primary key. 6 | This is a meta-backend which implements [Tuple store](tuple-strict.md). 7 | 8 | ## Supported backends 9 | 10 | * [MySQL](https://www.mysql.com) 11 | * [PostgreSQL](https://www.postgresql.org) 12 | -------------------------------------------------------------------------------- /docs/tuple-strict.md: -------------------------------------------------------------------------------- 1 | # Strict tuple store 2 | 3 | [![Go Reference](https://pkg.go.dev/badge/github.com/hidal-go/hidalgo/tuple.svg "GoDoc for strict tuple store within HiDAL-Go")](https://pkg.go.dev/github.com/hidal-go/hidalgo/tuple) 4 | 5 | This abstraction is based on tuple stores with a predefined schema. It should be able to isolate tuples of one type 6 | into separate tables, and support lookup and scans on primary key. No secondary keys are supported. 7 | 8 | By creating a simple key-value tables, these stores can implement [hierarchical key-value store](kv-hierarchical.md). 9 | 10 | ## Supported backends 11 | 12 | * [SQL](sql-tuple.md) (meta-backend) 13 | * [Google Datastore](https://cloud.google.com/datastore/) 14 | 15 | * Emulated over [Hierarchical KV](kv-hierarchical.md) 16 | -------------------------------------------------------------------------------- /filter/filters.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/hidal-go/hidalgo/values" 7 | ) 8 | 9 | type ValueFilter interface { 10 | FilterValue(v values.Value) bool 11 | } 12 | 13 | type SortableFilter interface { 14 | ValueFilter 15 | FilterSortable(v values.Sortable) bool 16 | // ValuesRange returns an optional range of value that matches the filter. 17 | // It is used as an optimization for complex filters for backend to limit the range of keys that will be considered. 18 | ValuesRange() *Range 19 | } 20 | 21 | var _ SortableFilter = Any{} 22 | 23 | type Any struct{} 24 | 25 | func (Any) FilterValue(v values.Value) bool { 26 | return v != nil 27 | } 28 | 29 | func (Any) FilterSortable(v values.Sortable) bool { 30 | return v != nil 31 | } 32 | 33 | func (Any) ValuesRange() *Range { 34 | return nil 35 | } 36 | 37 | // EQ is a shorthand for Equal. 38 | func EQ(v values.Value) SortableFilter { 39 | return Equal{Value: v} 40 | } 41 | 42 | var _ SortableFilter = Equal{} 43 | 44 | type Equal struct { 45 | Value values.Value 46 | } 47 | 48 | func (f Equal) FilterValue(a values.Value) bool { 49 | switch a := a.(type) { 50 | case values.Bytes: 51 | b, ok := f.Value.(values.Bytes) 52 | if !ok { 53 | return false 54 | } 55 | return bytes.Equal(a, b) 56 | } 57 | return f.Value == a 58 | } 59 | 60 | func (f Equal) FilterSortable(a values.Sortable) bool { 61 | b, ok := f.Value.(values.Sortable) 62 | if !ok { 63 | return a == nil && f.Value == nil 64 | } 65 | switch a := a.(type) { 66 | case values.Bytes: 67 | b, ok := b.(values.Bytes) 68 | if !ok { 69 | return false 70 | } 71 | return bytes.Equal(a, b) 72 | } 73 | return f.Value == a 74 | } 75 | 76 | func (f Equal) ValuesRange() *Range { 77 | b, ok := f.Value.(values.Sortable) 78 | if !ok { 79 | return nil 80 | } 81 | return &Range{ 82 | Start: GTE(b), 83 | End: LTE(b), 84 | } 85 | } 86 | 87 | // LT is a "less than" filter. Shorthand for Less. 88 | func LT(v values.Sortable) *Less { 89 | return &Less{Value: v} 90 | } 91 | 92 | // LTE is a "less than or equal" filter. Shorthand for Less. 93 | func LTE(v values.Sortable) *Less { 94 | return &Less{Value: v, Equal: true} 95 | } 96 | 97 | var _ SortableFilter = Less{} 98 | 99 | type Less struct { 100 | Value values.Sortable 101 | Equal bool 102 | } 103 | 104 | func (f Less) FilterValue(v values.Value) bool { 105 | a, ok := v.(values.Sortable) 106 | if !ok && v != nil { 107 | return false 108 | } 109 | return f.FilterSortable(a) 110 | } 111 | 112 | func (f Less) FilterSortable(v values.Sortable) bool { 113 | if v == nil { 114 | return true 115 | } 116 | c := values.Compare(v, f.Value) 117 | return c == -1 || (f.Equal && c == 0) 118 | } 119 | 120 | func (f Less) ValuesRange() *Range { 121 | return &Range{End: &f} 122 | } 123 | 124 | // GT is a "greater than" filter. Shorthand for Greater. 125 | func GT(v values.Sortable) *Greater { 126 | return &Greater{Value: v} 127 | } 128 | 129 | // GTE is a "greater than or equal" filter. Shorthand for Greater. 130 | func GTE(v values.Sortable) *Greater { 131 | return &Greater{Value: v, Equal: true} 132 | } 133 | 134 | var _ SortableFilter = Greater{} 135 | 136 | type Greater struct { 137 | Value values.Sortable 138 | Equal bool 139 | } 140 | 141 | func (f Greater) FilterValue(v values.Value) bool { 142 | a, ok := v.(values.Sortable) 143 | if !ok && v != nil { 144 | return false 145 | } 146 | return f.FilterSortable(a) 147 | } 148 | 149 | func (f Greater) FilterSortable(v values.Sortable) bool { 150 | if v == nil { 151 | return true 152 | } 153 | c := values.Compare(v, f.Value) 154 | return c == +1 || (f.Equal && c == 0) 155 | } 156 | 157 | func (f Greater) ValuesRange() *Range { 158 | return &Range{Start: &f} 159 | } 160 | 161 | var _ SortableFilter = Range{} 162 | 163 | // Range represents a range of sortable values. 164 | // If inclusive is set, the range is [start, end], if not, the range is (start, end). 165 | type Range struct { 166 | Start *Greater 167 | End *Less 168 | } 169 | 170 | // isPrefix checks if the range describes a prefix. In this case Start.Value describes the prefix. 171 | func (f Range) isPrefix() bool { 172 | if f.Start == nil || !f.Start.Equal { 173 | return false 174 | } 175 | s, ok := f.Start.Value.(values.BinaryString) 176 | if !ok { 177 | return false 178 | } 179 | end := s.PrefixEnd() 180 | if end == nil { 181 | return f.End == nil 182 | } 183 | if f.End == nil || f.End.Equal { 184 | return false 185 | } 186 | return values.Compare(end, f.End.Value) == 0 187 | } 188 | 189 | // Prefix returns a common prefix of the range. Boolean flag indicates if prefix fully describes the range. 190 | func (f Range) Prefix() (values.BinaryString, bool) { 191 | if !f.isPrefix() { 192 | // TODO: calculate common prefix 193 | return nil, false 194 | } 195 | p, ok := f.Start.Value.(values.BinaryString) 196 | return p, ok 197 | } 198 | 199 | func (f Range) FilterValue(v values.Value) bool { 200 | a, ok := v.(values.Sortable) 201 | if !ok && v != nil { 202 | return false 203 | } 204 | return f.FilterSortable(a) 205 | } 206 | 207 | func (f Range) FilterSortable(v values.Sortable) bool { 208 | if v == nil { 209 | return f.Start != nil 210 | } 211 | if f.Start != nil && !f.Start.FilterSortable(v) { 212 | return false 213 | } 214 | if f.End != nil && !f.End.FilterSortable(v) { 215 | return false 216 | } 217 | return true 218 | } 219 | 220 | func (f Range) ValuesRange() *Range { 221 | return &f 222 | } 223 | 224 | type And []ValueFilter 225 | 226 | func (arr And) FilterValue(v values.Value) bool { 227 | for _, f := range arr { 228 | if !f.FilterValue(v) { 229 | return false 230 | } 231 | } 232 | return true 233 | } 234 | 235 | type Or []ValueFilter 236 | 237 | func (arr Or) FilterValue(v values.Value) bool { 238 | for _, f := range arr { 239 | if f.FilterValue(v) { 240 | return true 241 | } 242 | } 243 | return false 244 | } 245 | 246 | type Not struct { 247 | Filter ValueFilter 248 | } 249 | 250 | func (f Not) FilterValue(v values.Value) bool { 251 | return !f.Filter.FilterValue(v) 252 | } 253 | 254 | func Prefix(pref values.BinaryString) SortableFilter { 255 | gt := GTE(pref) 256 | end := pref.PrefixEnd() 257 | if end == nil { 258 | return *gt 259 | } 260 | return Range{ 261 | Start: gt, 262 | End: LT(end), 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/hidal-go/hidalgo 2 | 3 | go 1.17 4 | 5 | require ( 6 | cloud.google.com/go/datastore v1.6.0 7 | github.com/boltdb/bolt v1.3.1 8 | github.com/cockroachdb/pebble v0.0.0-20220318150003-0ad186894f6d 9 | github.com/dgraph-io/badger/v2 v2.2007.4 10 | github.com/go-kivik/couchdb v2.0.0+incompatible 11 | github.com/go-kivik/kivik v2.0.0+incompatible 12 | github.com/go-kivik/pouchdb v2.0.1+incompatible 13 | github.com/go-sql-driver/mysql v1.6.0 14 | github.com/gogo/protobuf v1.3.2 15 | github.com/lib/pq v1.10.4 16 | github.com/ory/dockertest v3.3.5+incompatible 17 | github.com/pborman/uuid v1.2.1 18 | github.com/stretchr/testify v1.7.1 19 | github.com/syndtr/goleveldb v1.0.0 20 | go.etcd.io/bbolt v1.3.6 21 | go.mongodb.org/mongo-driver v1.8.4 22 | google.golang.org/api v0.73.0 23 | gopkg.in/olivere/elastic.v5 v5.0.86 24 | ) 25 | 26 | require ( 27 | cloud.google.com/go v0.100.2 // indirect 28 | cloud.google.com/go/compute v1.5.0 // indirect 29 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect 30 | github.com/DataDog/zstd v1.5.0 // indirect 31 | github.com/Microsoft/go-winio v0.5.2 // indirect 32 | github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect 33 | github.com/cenkalti/backoff v2.2.1+incompatible // indirect 34 | github.com/cespare/xxhash v1.1.0 // indirect 35 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 36 | github.com/cockroachdb/errors v1.9.0 // indirect 37 | github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect 38 | github.com/cockroachdb/redact v1.1.3 // indirect 39 | github.com/containerd/continuity v0.2.2 // indirect 40 | github.com/davecgh/go-spew v1.1.1 // indirect 41 | github.com/dgraph-io/ristretto v0.1.0 // indirect 42 | github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect 43 | github.com/docker/go-connections v0.4.0 // indirect 44 | github.com/docker/go-units v0.4.0 // indirect 45 | github.com/dustin/go-humanize v1.0.0 // indirect 46 | github.com/flimzy/diff v0.1.7 // indirect 47 | github.com/flimzy/testy v0.1.17 // indirect 48 | github.com/getsentry/sentry-go v0.13.0 // indirect 49 | github.com/go-kivik/kiviktest v2.0.0+incompatible // indirect 50 | github.com/go-stack/stack v1.8.1 // indirect 51 | github.com/golang/glog v1.0.0 // indirect 52 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 53 | github.com/golang/protobuf v1.5.2 // indirect 54 | github.com/golang/snappy v0.0.4 // indirect 55 | github.com/google/go-cmp v0.5.7 // indirect 56 | github.com/google/uuid v1.3.0 // indirect 57 | github.com/googleapis/gax-go/v2 v2.2.0 // indirect 58 | github.com/gopherjs/gopherjs v0.0.0-20220221023154-0b2280d3ff96 // indirect 59 | github.com/gopherjs/jsbuiltin v0.0.0-20180426082241-50091555e127 // indirect 60 | github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect 61 | github.com/josharian/intern v1.0.0 // indirect 62 | github.com/klauspost/compress v1.15.1 // indirect 63 | github.com/kr/pretty v0.3.0 // indirect 64 | github.com/kr/text v0.2.0 // indirect 65 | github.com/mailru/easyjson v0.7.7 // indirect 66 | github.com/opencontainers/go-digest v1.0.0 // indirect 67 | github.com/opencontainers/image-spec v1.0.2 // indirect 68 | github.com/opencontainers/runc v1.1.0 // indirect 69 | github.com/pkg/errors v0.9.1 // indirect 70 | github.com/pmezard/go-difflib v1.0.0 // indirect 71 | github.com/rogpeppe/go-internal v1.8.1 // indirect 72 | github.com/sirupsen/logrus v1.8.1 // indirect 73 | github.com/tidwall/pretty v1.2.0 // indirect 74 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 75 | github.com/xdg-go/scram v1.1.1 // indirect 76 | github.com/xdg-go/stringprep v1.0.3 // indirect 77 | github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect 78 | gitlab.com/flimzy/testy v0.10.2 // indirect 79 | go.opencensus.io v0.23.0 // indirect 80 | golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 // indirect 81 | golang.org/x/exp v0.0.0-20220321173239-a90fa8a75705 // indirect 82 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect 83 | golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect 84 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect 85 | golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect 86 | golang.org/x/text v0.3.7 // indirect 87 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 88 | google.golang.org/appengine v1.6.7 // indirect 89 | google.golang.org/genproto v0.0.0-20220317150908-0efb43f6373e // indirect 90 | google.golang.org/grpc v1.45.0 // indirect 91 | google.golang.org/protobuf v1.27.1 // indirect 92 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect 93 | gotest.tools v2.2.0+incompatible // indirect 94 | ) 95 | -------------------------------------------------------------------------------- /kv/all/all.go: -------------------------------------------------------------------------------- 1 | package all 2 | 3 | import ( 4 | _ "github.com/hidal-go/hidalgo/kv/bbolt" 5 | _ "github.com/hidal-go/hidalgo/kv/bolt" 6 | _ "github.com/hidal-go/hidalgo/kv/flat/all" 7 | ) 8 | -------------------------------------------------------------------------------- /kv/bbolt/bolt.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Cayley Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bbolt 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "time" 21 | 22 | bolt "go.etcd.io/bbolt" 23 | 24 | "github.com/hidal-go/hidalgo/base" 25 | "github.com/hidal-go/hidalgo/kv" 26 | ) 27 | 28 | const ( 29 | Name = "bbolt" 30 | ) 31 | 32 | func init() { 33 | kv.Register(kv.Registration{ 34 | Registration: base.Registration{ 35 | Name: Name, Title: "BBoltDB", 36 | Local: true, 37 | }, 38 | OpenPath: OpenPath, 39 | }) 40 | } 41 | 42 | var _ kv.KV = (*DB)(nil) 43 | 44 | func New(d *bolt.DB) *DB { 45 | return &DB{db: d} 46 | } 47 | 48 | func Open(path string, opt *bolt.Options) (*DB, error) { 49 | db, err := bolt.Open(path, 0o644, opt) 50 | if err != nil { 51 | return nil, err 52 | } 53 | return New(db), nil 54 | } 55 | 56 | func OpenPath(path string) (kv.KV, error) { 57 | db, err := Open(path, &bolt.Options{ 58 | Timeout: time.Second, 59 | }) 60 | if err != nil { 61 | return nil, err 62 | } 63 | return db, nil 64 | } 65 | 66 | type DB struct { 67 | db *bolt.DB 68 | } 69 | 70 | func (db *DB) DB() *bolt.DB { 71 | return db.db 72 | } 73 | 74 | func (db *DB) Close() error { 75 | return db.db.Close() 76 | } 77 | 78 | func (db *DB) Tx(ctx context.Context, rw bool) (kv.Tx, error) { 79 | tx, err := db.db.Begin(rw) 80 | if err != nil { 81 | return nil, err 82 | } 83 | return &Tx{tx: tx}, nil 84 | } 85 | 86 | func (db *DB) View(ctx context.Context, fn func(tx kv.Tx) error) error { 87 | return kv.View(ctx, db, fn) 88 | } 89 | 90 | func (db *DB) Update(ctx context.Context, fn func(tx kv.Tx) error) error { 91 | return kv.Update(ctx, db, fn) 92 | } 93 | 94 | type Tx struct { 95 | tx *bolt.Tx 96 | } 97 | 98 | func (tx *Tx) root() *bolt.Bucket { 99 | // a hack to get the root bucket 100 | c := tx.tx.Cursor() 101 | return c.Bucket() 102 | } 103 | 104 | func (tx *Tx) bucket(key kv.Key) (*bolt.Bucket, kv.Key) { 105 | if len(key) <= 1 { 106 | return tx.root(), key 107 | } 108 | b := tx.tx.Bucket(key[0]) 109 | key = key[1:] 110 | for b != nil && len(key) > 1 { 111 | b = b.Bucket(key[0]) 112 | key = key[1:] 113 | } 114 | return b, key 115 | } 116 | 117 | func (tx *Tx) Get(ctx context.Context, key kv.Key) (kv.Value, error) { 118 | b, k := tx.bucket(key) 119 | if b == nil || len(k) != 1 { 120 | return nil, kv.ErrNotFound 121 | } 122 | v := b.Get(k[0]) 123 | if v == nil { 124 | return nil, kv.ErrNotFound 125 | } 126 | return v, nil 127 | } 128 | 129 | func (tx *Tx) GetBatch(ctx context.Context, keys []kv.Key) ([]kv.Value, error) { 130 | vals := make([]kv.Value, len(keys)) 131 | for i, key := range keys { 132 | if b, k := tx.bucket(key); b != nil && len(k) == 1 { 133 | vals[i] = b.Get(k[0]) 134 | } 135 | } 136 | return vals, nil 137 | } 138 | 139 | func (tx *Tx) Commit(ctx context.Context) error { 140 | return tx.tx.Commit() 141 | } 142 | 143 | func (tx *Tx) Close() error { 144 | return tx.tx.Rollback() 145 | } 146 | 147 | func (tx *Tx) Put(ctx context.Context, k kv.Key, v kv.Value) error { 148 | var ( 149 | b *bolt.Bucket 150 | err error 151 | ) 152 | if len(k) <= 1 { 153 | b = tx.root() 154 | } else { 155 | b, err = tx.tx.CreateBucketIfNotExists(k[0]) 156 | k = k[1:] 157 | } 158 | for err == nil && b != nil && len(k) > 1 { 159 | b, err = b.CreateBucketIfNotExists(k[0]) 160 | k = k[1:] 161 | } 162 | if err != nil { 163 | return err 164 | } else if len(k[0]) == 0 && len(v) == 0 { 165 | return nil // bucket creation, no need to put value 166 | } 167 | err = b.Put(k[0], v) 168 | if err == bolt.ErrTxNotWritable { 169 | err = kv.ErrReadOnly 170 | } 171 | return err 172 | } 173 | 174 | func (tx *Tx) Del(ctx context.Context, k kv.Key) error { 175 | b, k := tx.bucket(k) 176 | if b == nil || len(k) != 1 { 177 | return nil 178 | } 179 | err := b.Delete(k[0]) 180 | if err == bolt.ErrTxNotWritable { 181 | err = kv.ErrReadOnly 182 | } 183 | return err 184 | } 185 | 186 | func (tx *Tx) Scan(ctx context.Context, opts ...kv.IteratorOption) kv.Iterator { 187 | var it kv.Iterator = &Iterator{ 188 | tx: tx, 189 | rootb: tx.root(), 190 | } 191 | it.Reset() 192 | return kv.ApplyIteratorOptions(it, opts) 193 | } 194 | 195 | var ( 196 | _ kv.Seeker = &Iterator{} 197 | _ kv.PrefixIterator = &Iterator{} 198 | ) 199 | 200 | type Iterator struct { 201 | tx *Tx // only used for iterator optimization 202 | rootb *bolt.Bucket 203 | rootk kv.Key // used to reconstruct a full key 204 | pref kv.Key // prefix to check all keys against 205 | stack struct { 206 | k kv.Key 207 | b []*bolt.Bucket 208 | c []*bolt.Cursor 209 | } 210 | k, v []byte // inside the current bucket 211 | } 212 | 213 | func (it *Iterator) Reset() { 214 | it.k = nil 215 | it.v = nil 216 | it.stack.c = nil 217 | it.stack.b = nil 218 | if cap(it.stack.k) >= len(it.rootk) { 219 | it.stack.k = it.stack.k[:len(it.rootk)] 220 | } else { 221 | // we will append to it 222 | it.stack.k = it.rootk.Clone() 223 | } 224 | copy(it.stack.k, it.rootk) 225 | if it.rootb != nil { 226 | it.stack.b = []*bolt.Bucket{it.rootb} 227 | } 228 | } 229 | 230 | func (it *Iterator) WithPrefix(pref kv.Key) kv.Iterator { 231 | it.Reset() 232 | kpref := pref 233 | b, p := it.tx.bucket(pref) 234 | if b == nil || len(p) > 1 { 235 | // if the prefix key is still longer than 1, it means that 236 | // a bucket mentioned in the prefix does not exists and 237 | // we can safely return an empty iterator 238 | *it = Iterator{tx: it.tx} 239 | return it 240 | } 241 | // the key for bucket we iterate 242 | it.rootk = kpref[:len(kpref)-len(p)] 243 | it.rootb = b 244 | it.pref = p 245 | it.Reset() 246 | return it 247 | } 248 | 249 | func (it *Iterator) next(pref kv.Key) bool { 250 | for len(it.stack.b) > 0 { 251 | i := len(it.stack.b) - 1 252 | cb := it.stack.b[i] 253 | if len(it.stack.c) < len(it.stack.b) { 254 | c := cb.Cursor() 255 | it.stack.c = append(it.stack.c, c) 256 | if i >= len(pref) { 257 | it.k, it.v = c.First() 258 | } else { 259 | it.k, it.v = c.Seek(pref[i]) 260 | } 261 | } else { 262 | c := it.stack.c[i] 263 | it.k, it.v = c.Next() 264 | } 265 | if it.k != nil { 266 | // found a key, check prefix 267 | if i >= len(pref) || bytes.HasPrefix(it.k, pref[i]) { 268 | // prefix matches, or is not specified 269 | if it.v == nil { 270 | // it's a bucket 271 | cb := it.stack.b[len(it.stack.b)-1] 272 | if b := cb.Bucket(it.k); b != nil { 273 | it.stack.b = append(it.stack.b, b) 274 | it.stack.k = append(it.stack.k, it.k) 275 | continue 276 | } 277 | // or maybe it's a key after all 278 | } 279 | // return this value 280 | return true 281 | } 282 | } 283 | // iterator is ended, or we reached the end of the prefix 284 | // return to top-level bucket 285 | it.stack.c = it.stack.c[:len(it.stack.c)-1] 286 | it.stack.b = it.stack.b[:len(it.stack.b)-1] 287 | if len(it.stack.k) > 0 { // since we hide top-level bucket it can be smaller 288 | it.stack.k = it.stack.k[:len(it.stack.k)-1] 289 | } 290 | } 291 | return false 292 | } 293 | 294 | func (it *Iterator) Seek(ctx context.Context, key kv.Key) bool { 295 | it.Reset() 296 | if !it.next(key) { 297 | return false 298 | } 299 | return it.Key().HasPrefix(it.pref) 300 | } 301 | 302 | func (it *Iterator) Next(ctx context.Context) bool { 303 | return it.next(it.pref) 304 | } 305 | 306 | func (it *Iterator) Key() kv.Key { 307 | if len(it.stack.b) == 0 { 308 | return nil 309 | } 310 | k := it.stack.k.Clone() 311 | k = append(k, append([]byte{}, it.k...)) 312 | return k 313 | } 314 | 315 | func (it *Iterator) Val() kv.Value { 316 | return it.v 317 | } 318 | 319 | func (it *Iterator) Err() error { 320 | return nil 321 | } 322 | 323 | func (it *Iterator) Close() error { 324 | *it = Iterator{} 325 | return nil 326 | } 327 | -------------------------------------------------------------------------------- /kv/bbolt/bolt_test.go: -------------------------------------------------------------------------------- 1 | package bbolt 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/hidal-go/hidalgo/kv" 8 | "github.com/hidal-go/hidalgo/kv/kvtest" 9 | ) 10 | 11 | func TestBBolt(t *testing.T) { 12 | kvtest.RunTestLocal(t, func(path string) (kv.KV, error) { 13 | path = filepath.Join(path, "bbolt.db") 14 | return OpenPath(path) 15 | }, nil) 16 | } 17 | -------------------------------------------------------------------------------- /kv/bolt/bolt.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Cayley Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package bolt 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "time" 21 | 22 | "github.com/boltdb/bolt" 23 | 24 | "github.com/hidal-go/hidalgo/base" 25 | "github.com/hidal-go/hidalgo/kv" 26 | ) 27 | 28 | const ( 29 | Name = "bolt" 30 | ) 31 | 32 | func init() { 33 | kv.Register(kv.Registration{ 34 | Registration: base.Registration{ 35 | Name: Name, Title: "BoltDB", 36 | Local: true, 37 | }, 38 | OpenPath: OpenPath, 39 | }) 40 | } 41 | 42 | var _ kv.KV = (*DB)(nil) 43 | 44 | func New(d *bolt.DB) *DB { 45 | return &DB{db: d} 46 | } 47 | 48 | func Open(path string, opt *bolt.Options) (*DB, error) { 49 | db, err := bolt.Open(path, 0o644, opt) 50 | if err != nil { 51 | return nil, err 52 | } 53 | return New(db), nil 54 | } 55 | 56 | func OpenPath(path string) (kv.KV, error) { 57 | db, err := Open(path, &bolt.Options{ 58 | Timeout: time.Second, 59 | }) 60 | if err != nil { 61 | return nil, err 62 | } 63 | return db, nil 64 | } 65 | 66 | type DB struct { 67 | db *bolt.DB 68 | } 69 | 70 | func (db *DB) DB() *bolt.DB { 71 | return db.db 72 | } 73 | 74 | func (db *DB) Close() error { 75 | return db.db.Close() 76 | } 77 | 78 | func (db *DB) Tx(ctx context.Context, rw bool) (kv.Tx, error) { 79 | tx, err := db.db.Begin(rw) 80 | if err != nil { 81 | return nil, err 82 | } 83 | return &Tx{tx: tx}, nil 84 | } 85 | 86 | func (db *DB) View(ctx context.Context, fn func(tx kv.Tx) error) error { 87 | return kv.View(ctx, db, fn) 88 | } 89 | 90 | func (db *DB) Update(ctx context.Context, fn func(tx kv.Tx) error) error { 91 | return kv.Update(ctx, db, fn) 92 | } 93 | 94 | type Tx struct { 95 | tx *bolt.Tx 96 | } 97 | 98 | func (tx *Tx) root() *bolt.Bucket { 99 | // a hack to get the root bucket 100 | c := tx.tx.Cursor() 101 | return c.Bucket() 102 | } 103 | 104 | func (tx *Tx) bucket(key kv.Key) (*bolt.Bucket, kv.Key) { 105 | if len(key) <= 1 { 106 | return tx.root(), key 107 | } 108 | b := tx.tx.Bucket(key[0]) 109 | key = key[1:] 110 | for b != nil && len(key) > 1 { 111 | b = b.Bucket(key[0]) 112 | key = key[1:] 113 | } 114 | return b, key 115 | } 116 | 117 | func (tx *Tx) Get(ctx context.Context, key kv.Key) (kv.Value, error) { 118 | b, k := tx.bucket(key) 119 | if b == nil || len(k) != 1 { 120 | return nil, kv.ErrNotFound 121 | } 122 | v := b.Get(k[0]) 123 | if v == nil { 124 | return nil, kv.ErrNotFound 125 | } 126 | return v, nil 127 | } 128 | 129 | func (tx *Tx) GetBatch(ctx context.Context, keys []kv.Key) ([]kv.Value, error) { 130 | vals := make([]kv.Value, len(keys)) 131 | for i, key := range keys { 132 | if b, k := tx.bucket(key); b != nil && len(k) == 1 { 133 | vals[i] = b.Get(k[0]) 134 | } 135 | } 136 | return vals, nil 137 | } 138 | 139 | func (tx *Tx) Commit(ctx context.Context) error { 140 | return tx.tx.Commit() 141 | } 142 | 143 | func (tx *Tx) Close() error { 144 | return tx.tx.Rollback() 145 | } 146 | 147 | func (tx *Tx) Put(ctx context.Context, k kv.Key, v kv.Value) error { 148 | var ( 149 | b *bolt.Bucket 150 | err error 151 | ) 152 | if len(k) <= 1 { 153 | b = tx.root() 154 | } else { 155 | b, err = tx.tx.CreateBucketIfNotExists(k[0]) 156 | k = k[1:] 157 | } 158 | for err == nil && b != nil && len(k) > 1 { 159 | b, err = b.CreateBucketIfNotExists(k[0]) 160 | k = k[1:] 161 | } 162 | if err != nil { 163 | return err 164 | } else if len(k[0]) == 0 && len(v) == 0 { 165 | return nil // bucket creation, no need to put value 166 | } 167 | err = b.Put(k[0], v) 168 | if err == bolt.ErrTxNotWritable { 169 | err = kv.ErrReadOnly 170 | } 171 | return err 172 | } 173 | 174 | func (tx *Tx) Del(ctx context.Context, k kv.Key) error { 175 | b, k := tx.bucket(k) 176 | if b == nil || len(k) != 1 { 177 | return nil 178 | } 179 | err := b.Delete(k[0]) 180 | if err == bolt.ErrTxNotWritable { 181 | err = kv.ErrReadOnly 182 | } 183 | return err 184 | } 185 | 186 | func (tx *Tx) Scan(ctx context.Context, opts ...kv.IteratorOption) kv.Iterator { 187 | var it kv.Iterator = &Iterator{ 188 | tx: tx, 189 | rootb: tx.root(), 190 | } 191 | it.Reset() 192 | return kv.ApplyIteratorOptions(it, opts) 193 | } 194 | 195 | var ( 196 | _ kv.Seeker = &Iterator{} 197 | _ kv.PrefixIterator = &Iterator{} 198 | ) 199 | 200 | type Iterator struct { 201 | tx *Tx // only used for iterator optimization 202 | rootb *bolt.Bucket 203 | rootk kv.Key // used to reconstruct a full key 204 | pref kv.Key // prefix to check all keys against 205 | stack struct { 206 | k kv.Key 207 | b []*bolt.Bucket 208 | c []*bolt.Cursor 209 | } 210 | k, v []byte // inside the current bucket 211 | } 212 | 213 | func (it *Iterator) Reset() { 214 | it.k = nil 215 | it.v = nil 216 | it.stack.c = nil 217 | it.stack.b = nil 218 | if cap(it.stack.k) >= len(it.rootk) { 219 | it.stack.k = it.stack.k[:len(it.rootk)] 220 | } else { 221 | // we will append to it 222 | it.stack.k = it.rootk.Clone() 223 | } 224 | copy(it.stack.k, it.rootk) 225 | if it.rootb != nil { 226 | it.stack.b = []*bolt.Bucket{it.rootb} 227 | } 228 | } 229 | 230 | func (it *Iterator) WithPrefix(pref kv.Key) kv.Iterator { 231 | it.Reset() 232 | kpref := pref 233 | b, p := it.tx.bucket(pref) 234 | if b == nil || len(p) > 1 { 235 | // if the prefix key is still longer than 1, it means that 236 | // a bucket mentioned in the prefix does not exists and 237 | // we can safely return an empty iterator 238 | *it = Iterator{tx: it.tx} 239 | return it 240 | } 241 | // the key for bucket we iterate 242 | it.rootk = kpref[:len(kpref)-len(p)] 243 | it.rootb = b 244 | it.pref = p 245 | it.Reset() 246 | return it 247 | } 248 | 249 | func (it *Iterator) next(pref kv.Key) bool { 250 | for len(it.stack.b) > 0 { 251 | i := len(it.stack.b) - 1 252 | cb := it.stack.b[i] 253 | if len(it.stack.c) < len(it.stack.b) { 254 | c := cb.Cursor() 255 | it.stack.c = append(it.stack.c, c) 256 | if i >= len(pref) { 257 | it.k, it.v = c.First() 258 | } else { 259 | it.k, it.v = c.Seek(pref[i]) 260 | } 261 | } else { 262 | c := it.stack.c[i] 263 | it.k, it.v = c.Next() 264 | } 265 | if it.k != nil { 266 | // found a key, check prefix 267 | if i >= len(pref) || bytes.HasPrefix(it.k, pref[i]) { 268 | // prefix matches, or is not specified 269 | if it.v == nil { 270 | // it's a bucket 271 | cb := it.stack.b[len(it.stack.b)-1] 272 | if b := cb.Bucket(it.k); b != nil { 273 | it.stack.b = append(it.stack.b, b) 274 | it.stack.k = append(it.stack.k, it.k) 275 | continue 276 | } 277 | // or maybe it's a key after all 278 | } 279 | // return this value 280 | return true 281 | } 282 | } 283 | // iterator is ended, or we reached the end of the prefix 284 | // return to top-level bucket 285 | it.stack.c = it.stack.c[:len(it.stack.c)-1] 286 | it.stack.b = it.stack.b[:len(it.stack.b)-1] 287 | if len(it.stack.k) > 0 { // since we hide top-level bucket it can be smaller 288 | it.stack.k = it.stack.k[:len(it.stack.k)-1] 289 | } 290 | } 291 | return false 292 | } 293 | 294 | func (it *Iterator) Seek(ctx context.Context, key kv.Key) bool { 295 | it.Reset() 296 | if !it.next(key) { 297 | return false 298 | } 299 | return it.Key().HasPrefix(it.pref) 300 | } 301 | 302 | func (it *Iterator) Next(ctx context.Context) bool { 303 | return it.next(it.pref) 304 | } 305 | 306 | func (it *Iterator) Key() kv.Key { 307 | if len(it.stack.b) == 0 { 308 | return nil 309 | } 310 | k := it.stack.k.Clone() 311 | k = append(k, append([]byte{}, it.k...)) 312 | return k 313 | } 314 | 315 | func (it *Iterator) Val() kv.Value { 316 | return it.v 317 | } 318 | 319 | func (it *Iterator) Err() error { 320 | return nil 321 | } 322 | 323 | func (it *Iterator) Close() error { 324 | *it = Iterator{} 325 | return nil 326 | } 327 | -------------------------------------------------------------------------------- /kv/bolt/bolt_test.go: -------------------------------------------------------------------------------- 1 | package bolt 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/hidal-go/hidalgo/kv" 8 | "github.com/hidal-go/hidalgo/kv/kvtest" 9 | ) 10 | 11 | func TestBolt(t *testing.T) { 12 | kvtest.RunTestLocal(t, func(path string) (kv.KV, error) { 13 | path = filepath.Join(path, "bolt.db") 14 | return OpenPath(path) 15 | }, nil) 16 | } 17 | -------------------------------------------------------------------------------- /kv/flat/all/all.go: -------------------------------------------------------------------------------- 1 | package all 2 | 3 | import ( 4 | _ "github.com/hidal-go/hidalgo/kv/flat/badger" 5 | _ "github.com/hidal-go/hidalgo/kv/flat/btree" 6 | _ "github.com/hidal-go/hidalgo/kv/flat/leveldb" 7 | ) 8 | -------------------------------------------------------------------------------- /kv/flat/all/all_64.go: -------------------------------------------------------------------------------- 1 | //go:build !386 && !arm 2 | 3 | package all 4 | 5 | // Backends that don't support 32bit 6 | 7 | import ( 8 | _ "github.com/hidal-go/hidalgo/kv/flat/pebble" 9 | ) 10 | -------------------------------------------------------------------------------- /kv/flat/badger/badger.go: -------------------------------------------------------------------------------- 1 | package badger 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/dgraph-io/badger/v2" 7 | 8 | "github.com/hidal-go/hidalgo/base" 9 | "github.com/hidal-go/hidalgo/kv/flat" 10 | ) 11 | 12 | const ( 13 | Name = "badger" 14 | ) 15 | 16 | func init() { 17 | flat.Register(flat.Registration{ 18 | Registration: base.Registration{ 19 | Name: Name, Title: "Badger", 20 | Local: true, 21 | }, 22 | OpenPath: OpenPath, 23 | }) 24 | } 25 | 26 | var _ flat.KV = (*DB)(nil) 27 | 28 | func New(d *badger.DB) *DB { 29 | return &DB{db: d} 30 | } 31 | 32 | func Open(opt badger.Options) (*DB, error) { 33 | if opt.ValueDir == "" { 34 | opt.ValueDir = opt.Dir 35 | } 36 | db, err := badger.Open(opt) 37 | if err != nil { 38 | return nil, err 39 | } 40 | return New(db), nil 41 | } 42 | 43 | func OpenPath(path string) (flat.KV, error) { 44 | opt := badger.DefaultOptions(path) 45 | db, err := Open(opt) 46 | if err != nil { 47 | return nil, err 48 | } 49 | return db, nil 50 | } 51 | 52 | type DB struct { 53 | db *badger.DB 54 | closed bool 55 | } 56 | 57 | func (db *DB) DB() *badger.DB { 58 | return db.db 59 | } 60 | 61 | func (db *DB) Close() error { 62 | if db.closed { 63 | return nil 64 | } 65 | db.closed = true 66 | return db.db.Close() 67 | } 68 | 69 | func (db *DB) Tx(ctx context.Context, rw bool) (flat.Tx, error) { 70 | tx := db.db.NewTransaction(rw) 71 | return &Tx{tx: tx}, nil 72 | } 73 | 74 | func (db *DB) View(ctx context.Context, fn func(tx flat.Tx) error) error { 75 | return flat.View(ctx, db, fn) 76 | } 77 | 78 | func (db *DB) Update(ctx context.Context, fn func(tx flat.Tx) error) error { 79 | return flat.Update(ctx, db, fn) 80 | } 81 | 82 | type Tx struct { 83 | tx *badger.Txn 84 | } 85 | 86 | func (tx *Tx) Commit(ctx context.Context) error { 87 | err := tx.tx.Commit() 88 | if err == badger.ErrConflict { 89 | err = flat.ErrConflict 90 | } 91 | return err 92 | } 93 | 94 | func (tx *Tx) Close() error { 95 | tx.tx.Discard() 96 | return nil 97 | } 98 | 99 | func (tx *Tx) Get(ctx context.Context, key flat.Key) (flat.Value, error) { 100 | if len(key) == 0 { 101 | return nil, flat.ErrNotFound 102 | } 103 | item, err := tx.tx.Get(key) 104 | if err == badger.ErrKeyNotFound { 105 | return nil, flat.ErrNotFound 106 | } else if err != nil { 107 | return nil, err 108 | } 109 | return item.ValueCopy(nil) 110 | } 111 | 112 | func (tx *Tx) GetBatch(ctx context.Context, keys []flat.Key) ([]flat.Value, error) { 113 | return flat.GetBatch(ctx, tx, keys) 114 | } 115 | 116 | func (tx *Tx) Put(ctx context.Context, k flat.Key, v flat.Value) error { 117 | err := tx.tx.Set(k, v) 118 | if err == badger.ErrConflict { 119 | err = flat.ErrConflict 120 | } 121 | return err 122 | } 123 | 124 | func (tx *Tx) Del(ctx context.Context, k flat.Key) error { 125 | err := tx.tx.Delete(k) 126 | if err == badger.ErrConflict { 127 | err = flat.ErrConflict 128 | } 129 | return err 130 | } 131 | 132 | func (tx *Tx) Scan(ctx context.Context, opts ...flat.IteratorOption) flat.Iterator { 133 | bit := tx.tx.NewIterator(badger.DefaultIteratorOptions) 134 | var it flat.Iterator = &Iterator{it: bit, first: true} 135 | it = flat.ApplyIteratorOptions(it, opts) 136 | return it 137 | } 138 | 139 | var ( 140 | _ flat.Seeker = &Iterator{} 141 | _ flat.PrefixIterator = &Iterator{} 142 | ) 143 | 144 | type Iterator struct { 145 | it *badger.Iterator 146 | pref flat.Key 147 | first bool 148 | valid bool 149 | err error 150 | } 151 | 152 | func (it *Iterator) Reset() { 153 | it.first = true 154 | it.valid = false 155 | it.err = nil 156 | } 157 | 158 | func (it *Iterator) WithPrefix(pref flat.Key) flat.Iterator { 159 | it.Reset() 160 | it.pref = pref 161 | return it 162 | } 163 | 164 | func (it *Iterator) next() bool { 165 | if len(it.pref) != 0 { 166 | it.valid = it.it.ValidForPrefix(it.pref) 167 | } else { 168 | it.valid = it.it.Valid() 169 | } 170 | return it.valid 171 | } 172 | 173 | func (it *Iterator) Seek(ctx context.Context, key flat.Key) bool { 174 | it.Reset() 175 | it.first = false 176 | it.it.Seek(key) 177 | return it.next() 178 | } 179 | 180 | func (it *Iterator) Next(ctx context.Context) bool { 181 | if it.first { 182 | it.first = false 183 | it.it.Seek(it.pref) 184 | } else { 185 | it.it.Next() 186 | } 187 | return it.next() 188 | } 189 | 190 | func (it *Iterator) Err() error { 191 | return it.err 192 | } 193 | 194 | func (it *Iterator) Close() error { 195 | it.it.Close() 196 | return it.Err() 197 | } 198 | 199 | func (it *Iterator) Key() flat.Key { 200 | if !it.valid { 201 | return nil 202 | } 203 | ent := it.it.Item() 204 | if ent == nil { 205 | return nil 206 | } 207 | return ent.Key() 208 | } 209 | 210 | func (it *Iterator) Val() flat.Value { 211 | if !it.valid { 212 | return nil 213 | } 214 | ent := it.it.Item() 215 | if ent == nil { 216 | return nil 217 | } 218 | v, err := ent.ValueCopy(nil) 219 | if err != nil { 220 | it.err = err 221 | } 222 | return v 223 | } 224 | -------------------------------------------------------------------------------- /kv/flat/badger/badger_test.go: -------------------------------------------------------------------------------- 1 | package badger 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/hidal-go/hidalgo/kv/flat" 7 | "github.com/hidal-go/hidalgo/kv/kvtest" 8 | ) 9 | 10 | func TestBadger(t *testing.T) { 11 | kvtest.RunTestLocal(t, flat.UpgradeOpenPath(OpenPath), nil) 12 | } 13 | -------------------------------------------------------------------------------- /kv/flat/btree/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2014 The Cayley Authors. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | .PHONY: specify 16 | 17 | # Do not commit changes to this line unless you are satisfied 18 | # that the github.com/cznic/b tests AND project tests 19 | pinned=82d9e96a4503a42315b0fdf5201314302beafe06 20 | 21 | specify: 22 | rm -rf b 23 | git clone https://github.com/cznic/b 24 | cd b && git checkout $(pinned) 25 | go test ./b 26 | @sed -e 's|interface{}[^{]*/\*K\*/|\[\]byte|g' -e 's|interface{}[^{]*/\*V\*/|\[\]byte|g' b/btree.go >keys.go 27 | rm -rf b 28 | -------------------------------------------------------------------------------- /kv/flat/btree/btree.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Cayley Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package btree 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "io" 21 | 22 | "github.com/hidal-go/hidalgo/base" 23 | "github.com/hidal-go/hidalgo/kv/flat" 24 | ) 25 | 26 | const ( 27 | Name = "btree" 28 | ) 29 | 30 | func init() { 31 | flat.Register(flat.Registration{ 32 | Registration: base.Registration{ 33 | Name: Name, Title: "B-Tree", 34 | Local: true, Volatile: true, 35 | }, 36 | OpenPath: func(path string) (flat.KV, error) { 37 | if path != "" { 38 | return nil, base.ErrVolatile 39 | } 40 | return New(), nil 41 | }, 42 | }) 43 | } 44 | 45 | var _ flat.KV = (*DB)(nil) 46 | 47 | // New creates a new flat in-memory key-value store. 48 | // It's not safe for concurrent use. 49 | func New() *DB { 50 | return &DB{t: TreeNew(bytes.Compare)} 51 | } 52 | 53 | type DB struct { 54 | t *Tree 55 | } 56 | 57 | func (db *DB) Close() error { 58 | return nil 59 | } 60 | 61 | func (db *DB) Tx(ctx context.Context, rw bool) (flat.Tx, error) { 62 | return &Tx{t: db.t, rw: rw}, nil 63 | } 64 | 65 | func (db *DB) View(ctx context.Context, fn func(tx flat.Tx) error) error { 66 | return flat.View(ctx, db, fn) 67 | } 68 | 69 | func (db *DB) Update(ctx context.Context, fn func(tx flat.Tx) error) error { 70 | return flat.Update(ctx, db, fn) 71 | } 72 | 73 | type Tx struct { 74 | t *Tree 75 | rw bool 76 | } 77 | 78 | func (tx *Tx) Get(ctx context.Context, key flat.Key) (flat.Value, error) { 79 | v, ok := tx.t.Get(key) 80 | if !ok { 81 | return nil, flat.ErrNotFound 82 | } 83 | return flat.Value(v).Clone(), nil 84 | } 85 | 86 | func (tx *Tx) GetBatch(ctx context.Context, keys []flat.Key) ([]flat.Value, error) { 87 | vals := make([]flat.Value, len(keys)) 88 | for i, k := range keys { 89 | if v, ok := tx.t.Get(k); ok { 90 | vals[i] = flat.Value(v).Clone() 91 | } 92 | } 93 | return vals, nil 94 | } 95 | 96 | func (tx *Tx) Commit(ctx context.Context) error { 97 | return nil 98 | } 99 | 100 | func (tx *Tx) Close() error { 101 | return nil 102 | } 103 | 104 | func (tx *Tx) Put(ctx context.Context, k flat.Key, v flat.Value) error { 105 | if !tx.rw { 106 | return flat.ErrReadOnly 107 | } 108 | tx.t.Set(k.Clone(), v.Clone()) 109 | return nil 110 | } 111 | 112 | func (tx *Tx) Del(ctx context.Context, k flat.Key) error { 113 | if !tx.rw { 114 | return flat.ErrReadOnly 115 | } 116 | tx.t.Delete(k) 117 | return nil 118 | } 119 | 120 | func (tx *Tx) Scan(ctx context.Context, opts ...flat.IteratorOption) flat.Iterator { 121 | var it flat.Iterator = &Iterator{t: tx.t} 122 | it = flat.ApplyIteratorOptions(it, opts) 123 | return it 124 | } 125 | 126 | var ( 127 | _ flat.Seeker = &Iterator{} 128 | _ flat.PrefixIterator = &Iterator{} 129 | ) 130 | 131 | type Iterator struct { 132 | t *Tree 133 | pref []byte 134 | e *Enumerator 135 | k, v []byte 136 | } 137 | 138 | func (it *Iterator) Reset() { 139 | if it.e != nil { 140 | it.e.Close() 141 | it.e = nil 142 | } 143 | it.k = nil 144 | it.v = nil 145 | } 146 | 147 | func (it *Iterator) WithPrefix(pref flat.Key) flat.Iterator { 148 | it.Reset() 149 | it.pref = pref 150 | return it 151 | } 152 | 153 | func (it *Iterator) next() bool { 154 | k, v, err := it.e.Next() 155 | if err == io.EOF { 156 | return false 157 | } else if !bytes.HasPrefix(k, it.pref) { 158 | return false 159 | } 160 | it.k, it.v = k, v 161 | return true 162 | } 163 | 164 | func (it *Iterator) Seek(ctx context.Context, key flat.Key) bool { 165 | if it.t == nil { 166 | return false 167 | } 168 | it.Reset() 169 | it.e, _ = it.t.Seek(key) 170 | return it.next() 171 | } 172 | 173 | func (it *Iterator) Next(ctx context.Context) bool { 174 | if it.t == nil { 175 | return false 176 | } 177 | if it.e == nil { 178 | it.e, _ = it.t.Seek(it.pref) 179 | } 180 | return it.next() 181 | } 182 | 183 | func (it *Iterator) Key() flat.Key { return it.k } 184 | func (it *Iterator) Val() flat.Value { return it.v } 185 | func (it *Iterator) Err() error { return nil } 186 | 187 | func (it *Iterator) Close() error { 188 | if it.e != nil { 189 | it.e.Close() 190 | it.e = nil 191 | } 192 | return it.Err() 193 | } 194 | -------------------------------------------------------------------------------- /kv/flat/btree/btree_test.go: -------------------------------------------------------------------------------- 1 | package btree 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/hidal-go/hidalgo/kv" 7 | "github.com/hidal-go/hidalgo/kv/flat" 8 | "github.com/hidal-go/hidalgo/kv/kvtest" 9 | ) 10 | 11 | func TestBtree(t *testing.T) { 12 | kvtest.RunTest(t, func(t testing.TB) kv.KV { 13 | return flat.Upgrade(New()) 14 | }, &kvtest.Options{ 15 | NoLocks: true, 16 | NoTx: true, 17 | }) 18 | } 19 | -------------------------------------------------------------------------------- /kv/flat/helpers.go: -------------------------------------------------------------------------------- 1 | package flat 2 | 3 | import "context" 4 | 5 | // Update is a helper to open a read-write transaction and update the database. 6 | // The update function may be called multiple times in case of conflicts with other writes. 7 | func Update(ctx context.Context, kv KV, update func(tx Tx) error) error { 8 | for { 9 | err := func() error { 10 | tx, err := kv.Tx(ctx, true) 11 | if err != nil { 12 | return err 13 | } 14 | defer tx.Close() 15 | err = update(tx) 16 | if err != nil { 17 | return err 18 | } 19 | return tx.Commit(ctx) 20 | }() 21 | if err == ErrConflict { 22 | continue 23 | } else if err != nil { 24 | return err 25 | } 26 | return nil 27 | } 28 | } 29 | 30 | // View is a helper to open a read-only transaction to read the database. 31 | func View(ctx context.Context, kv KV, view func(tx Tx) error) error { 32 | tx, err := kv.Tx(ctx, false) 33 | if err != nil { 34 | return err 35 | } 36 | if err = view(tx); err != nil { 37 | defer tx.Close() 38 | return err 39 | } 40 | return tx.Close() 41 | } 42 | 43 | // Each is a helper to enumerate all key-value pairs with a specific prefix. 44 | // See Iterator for rules of using returned values. 45 | func Each(ctx context.Context, tx Tx, fnc func(k Key, v Value) error, opts ...IteratorOption) error { 46 | it := tx.Scan(ctx, opts...) 47 | defer it.Close() 48 | for it.Next(ctx) { 49 | if err := fnc(it.Key(), it.Val()); err != nil { 50 | return err 51 | } 52 | } 53 | return it.Err() 54 | } 55 | -------------------------------------------------------------------------------- /kv/flat/iterator.go: -------------------------------------------------------------------------------- 1 | package flat 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | 7 | "github.com/hidal-go/hidalgo/base" 8 | ) 9 | 10 | // Iterator is an iterator over hierarchical key-value store. 11 | type Iterator interface { 12 | base.Iterator 13 | // Reset the iterator to the starting state. Closed iterator can not reset. 14 | Reset() 15 | // Key return current key. Returned value will become invalid on Next or Close. 16 | // Caller should not modify or store the value - use Clone. 17 | Key() Key 18 | // Val return current value. Returned value will become invalid on Next or Close. 19 | // Caller should not modify or store the value - use Clone. 20 | Val() Value 21 | } 22 | 23 | type Seeker interface { 24 | Iterator 25 | // Seek the iterator to a given key. If the key does not exist, the next key is used. 26 | // Function returns false if there is no key greater or equal to a given one. 27 | Seek(ctx context.Context, key Key) bool 28 | } 29 | 30 | // Seek the iterator to a given key. If the key does not exist, the next key is used. 31 | // Function returns false if there is no key greater or equal to a given one. 32 | func Seek(ctx context.Context, it Iterator, key Key) bool { 33 | if it, ok := it.(Seeker); ok { 34 | return it.Seek(ctx, key) 35 | } 36 | if len(key) == 0 { 37 | it.Reset() // seek to the beginning 38 | return it.Next(ctx) 39 | } 40 | // check where we currently are 41 | k := it.Key() 42 | if len(k) == 0 { 43 | // might either be the beginning, or the end, so restart to be sure 44 | it.Reset() 45 | } else { 46 | switch bytes.Compare(k, key) { 47 | case 0: 48 | return true // already there 49 | case -1: 50 | // can seek forward without resetting 51 | case +1: 52 | it.Reset() // too far, must restart 53 | } 54 | } 55 | for it.Next(ctx) { 56 | k = it.Key() 57 | if bytes.Compare(k, key) >= 0 { 58 | return true 59 | } 60 | } 61 | return false 62 | } 63 | 64 | // ApplyIteratorOptions applies all iterator options. 65 | func ApplyIteratorOptions(it Iterator, opts []IteratorOption) Iterator { 66 | for _, opt := range opts { 67 | it = opt.ApplyFlat(it) 68 | } 69 | return it 70 | } 71 | 72 | // IteratorOption is an additional option that affects iterator behaviour. 73 | // 74 | // Implementations in generic KV package should assert for an optimized version of option (via interface assertion), 75 | // and fallback to generic implementation if the store doesn't support this option natively. 76 | type IteratorOption interface { 77 | // ApplyFlat option to the flat KV iterator. Implementation may wrap or replace the iterator. 78 | ApplyFlat(it Iterator) Iterator 79 | } 80 | 81 | // IteratorOptionFunc is a function type that implements IteratorOption. 82 | type IteratorOptionFunc func(it Iterator) Iterator 83 | 84 | func (opt IteratorOptionFunc) ApplyFlat(it Iterator) Iterator { 85 | return opt(it) 86 | } 87 | 88 | // PrefixIterator is an Iterator optimization to support WithPrefix option. 89 | type PrefixIterator interface { 90 | Iterator 91 | // WithPrefix implements WithPrefix iterator option. 92 | // Current iterator will be replaced with a new one and must not be used after this call. 93 | WithPrefix(pref Key) Iterator 94 | } 95 | -------------------------------------------------------------------------------- /kv/flat/kv.go: -------------------------------------------------------------------------------- 1 | // Package flat provides an abstraction over flat key-value stores. 2 | package flat 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | 8 | "github.com/hidal-go/hidalgo/base" 9 | "github.com/hidal-go/hidalgo/kv" 10 | ) 11 | 12 | var ( 13 | // ErrNotFound is returned then a key was not found in the database. 14 | ErrNotFound = kv.ErrNotFound 15 | // ErrReadOnly is returned when write operation is performed on read-only database or transaction. 16 | ErrReadOnly = kv.ErrReadOnly 17 | // ErrConflict is returned when write operation performed be current transaction cannot be committed 18 | // because of another concurrent write. Caller must restart the transaction. 19 | ErrConflict = kv.ErrConflict 20 | ) 21 | 22 | // KV is an interface for flat key-value databases. 23 | type KV interface { 24 | base.DB 25 | Tx(ctx context.Context, rw bool) (Tx, error) 26 | View(ctx context.Context, fn func(tx Tx) error) error 27 | Update(ctx context.Context, fn func(tx Tx) error) error 28 | } 29 | 30 | // Key is a flat binary key used in a database. 31 | type Key []byte 32 | 33 | // Clone returns a copy of the key. 34 | func (k Key) Clone() Key { 35 | if k == nil { 36 | return nil 37 | } 38 | p := make(Key, len(k)) 39 | copy(p, k) 40 | return p 41 | } 42 | 43 | // Value is a binary value stored in a database. 44 | type Value = kv.Value 45 | 46 | // Pair is a key-value pair. 47 | type Pair struct { 48 | Key Key 49 | Val Value 50 | } 51 | 52 | func (p Pair) String() string { 53 | return fmt.Sprintf("%x = %x", p.Key, p.Val) 54 | } 55 | 56 | type Getter interface { 57 | // Get fetches a value for a single key from the database. 58 | // It return ErrNotFound if key does not exists. 59 | Get(ctx context.Context, key Key) (Value, error) 60 | } 61 | 62 | // Tx is a transaction over flat key-value store. 63 | type Tx interface { 64 | base.Tx 65 | Getter 66 | // GetBatch fetches values for multiple keys from the database. 67 | // Nil element in the slice indicates that key does not exists. 68 | GetBatch(ctx context.Context, keys []Key) ([]Value, error) 69 | // Put writes a key-value pair to the database. 70 | // New value will immediately be visible by Get on the same Tx, 71 | // but implementation might buffer the write until transaction is committed. 72 | Put(ctx context.Context, k Key, v Value) error 73 | // Del removes the key from the database. See Put for consistency guaranties. 74 | Del(ctx context.Context, k Key) error 75 | // Scan starts iteration over key-value pairs. Returned results are affected by IteratorOption. 76 | Scan(ctx context.Context, opts ...IteratorOption) Iterator 77 | } 78 | 79 | // GetBatch is an implementation of Tx.GetBatch for databases that has no native implementation for it. 80 | func GetBatch(ctx context.Context, tx Getter, keys []Key) ([]Value, error) { 81 | vals := make([]Value, len(keys)) 82 | var err error 83 | for i, k := range keys { 84 | vals[i], err = tx.Get(ctx, k) 85 | if err == ErrNotFound { 86 | vals[i] = nil 87 | } else if err != nil { 88 | return nil, err 89 | } 90 | } 91 | return vals, nil 92 | } 93 | -------------------------------------------------------------------------------- /kv/flat/leveldb/leveldb.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The Cayley Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package leveldb 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/syndtr/goleveldb/leveldb" 21 | "github.com/syndtr/goleveldb/leveldb/iterator" 22 | "github.com/syndtr/goleveldb/leveldb/opt" 23 | "github.com/syndtr/goleveldb/leveldb/util" 24 | 25 | "github.com/hidal-go/hidalgo/base" 26 | "github.com/hidal-go/hidalgo/kv/flat" 27 | ) 28 | 29 | const ( 30 | Name = "leveldb" 31 | ) 32 | 33 | func init() { 34 | flat.Register(flat.Registration{ 35 | Registration: base.Registration{ 36 | Name: Name, Title: "LevelDB", 37 | Local: true, 38 | }, 39 | OpenPath: OpenPath, 40 | }) 41 | } 42 | 43 | var _ flat.KV = (*DB)(nil) 44 | 45 | func New(d *leveldb.DB) *DB { 46 | return &DB{db: d} 47 | } 48 | 49 | func Open(path string, opt *opt.Options) (*DB, error) { 50 | db, err := leveldb.OpenFile(path, opt) 51 | if err != nil { 52 | return nil, err 53 | } 54 | return New(db), nil 55 | } 56 | 57 | func OpenPath(path string) (flat.KV, error) { 58 | db, err := Open(path, nil) 59 | if err != nil { 60 | return nil, err 61 | } 62 | return db, nil 63 | } 64 | 65 | type DB struct { 66 | db *leveldb.DB 67 | wo *opt.WriteOptions 68 | ro *opt.ReadOptions 69 | } 70 | 71 | func (db *DB) SetWriteOptions(wo *opt.WriteOptions) { 72 | db.wo = wo 73 | } 74 | 75 | func (db *DB) SetReadOptions(ro *opt.ReadOptions) { 76 | db.ro = ro 77 | } 78 | 79 | func (db *DB) Close() error { 80 | return db.db.Close() 81 | } 82 | 83 | func (db *DB) Tx(ctx context.Context, rw bool) (flat.Tx, error) { 84 | tx := &Tx{db: db} 85 | var err error 86 | if rw { 87 | tx.tx, err = db.db.OpenTransaction() 88 | } else { 89 | tx.sn, err = db.db.GetSnapshot() 90 | } 91 | if err != nil { 92 | return nil, err 93 | } 94 | return tx, nil 95 | } 96 | 97 | func (db *DB) View(ctx context.Context, fn func(tx flat.Tx) error) error { 98 | return flat.View(ctx, db, fn) 99 | } 100 | 101 | func (db *DB) Update(ctx context.Context, fn func(tx flat.Tx) error) error { 102 | return flat.Update(ctx, db, fn) 103 | } 104 | 105 | type Tx struct { 106 | db *DB 107 | sn *leveldb.Snapshot 108 | tx *leveldb.Transaction 109 | err error 110 | } 111 | 112 | func (tx *Tx) Commit(ctx context.Context) error { 113 | if tx.err != nil { 114 | return tx.err 115 | } 116 | if tx.tx != nil { 117 | tx.err = tx.tx.Commit() 118 | return tx.err 119 | } 120 | tx.sn.Release() 121 | return tx.err 122 | } 123 | 124 | func (tx *Tx) Close() error { 125 | if tx.tx != nil { 126 | tx.tx.Discard() 127 | } else { 128 | tx.sn.Release() 129 | } 130 | return tx.err 131 | } 132 | 133 | func (tx *Tx) Get(ctx context.Context, key flat.Key) (flat.Value, error) { 134 | var ( 135 | val []byte 136 | err error 137 | ) 138 | if tx.tx != nil { 139 | val, err = tx.tx.Get(key, tx.db.ro) 140 | } else { 141 | val, err = tx.sn.Get(key, tx.db.ro) 142 | } 143 | if err == leveldb.ErrNotFound { 144 | return nil, flat.ErrNotFound 145 | } else if err != nil { 146 | return nil, err 147 | } 148 | return val, nil 149 | } 150 | 151 | func (tx *Tx) GetBatch(ctx context.Context, keys []flat.Key) ([]flat.Value, error) { 152 | return flat.GetBatch(ctx, tx, keys) 153 | } 154 | 155 | func (tx *Tx) Put(ctx context.Context, k flat.Key, v flat.Value) error { 156 | if tx.tx == nil { 157 | return flat.ErrReadOnly 158 | } 159 | return tx.tx.Put(k, v, tx.db.wo) 160 | } 161 | 162 | func (tx *Tx) Del(ctx context.Context, k flat.Key) error { 163 | if tx.tx == nil { 164 | return flat.ErrReadOnly 165 | } 166 | return tx.tx.Delete(k, tx.db.wo) 167 | } 168 | 169 | func (tx *Tx) Scan(ctx context.Context, opts ...flat.IteratorOption) flat.Iterator { 170 | lit := &Iterator{tx: tx} 171 | lit.WithPrefix(nil) 172 | var it flat.Iterator = lit 173 | it = flat.ApplyIteratorOptions(it, opts) 174 | return it 175 | } 176 | 177 | var ( 178 | _ flat.Seeker = &Iterator{} 179 | _ flat.PrefixIterator = &Iterator{} 180 | ) 181 | 182 | type Iterator struct { 183 | tx *Tx 184 | it iterator.Iterator 185 | first bool 186 | } 187 | 188 | func (it *Iterator) Reset() { 189 | it.first = true 190 | } 191 | 192 | func (it *Iterator) WithPrefix(pref flat.Key) flat.Iterator { 193 | it.Reset() 194 | if it.it != nil { 195 | it.it.Release() 196 | } 197 | r, ro := util.BytesPrefix(pref), it.tx.db.ro 198 | if it.tx.tx != nil { 199 | it.it = it.tx.tx.NewIterator(r, ro) 200 | } else { 201 | it.it = it.tx.sn.NewIterator(r, ro) 202 | } 203 | return it 204 | } 205 | 206 | func (it *Iterator) Seek(ctx context.Context, key flat.Key) bool { 207 | it.Reset() 208 | it.first = false 209 | return it.it.Seek(key) 210 | } 211 | 212 | func (it *Iterator) Next(ctx context.Context) bool { 213 | if it.first { 214 | it.first = false 215 | return it.it.First() 216 | } 217 | return it.it.Next() 218 | } 219 | 220 | func (it *Iterator) Key() flat.Key { return it.it.Key() } 221 | func (it *Iterator) Val() flat.Value { return it.it.Value() } 222 | func (it *Iterator) Err() error { return it.it.Error() } 223 | 224 | func (it *Iterator) Close() error { 225 | it.it.Release() 226 | return it.Err() 227 | } 228 | -------------------------------------------------------------------------------- /kv/flat/leveldb/leveldb_test.go: -------------------------------------------------------------------------------- 1 | package leveldb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/hidal-go/hidalgo/kv/flat" 7 | "github.com/hidal-go/hidalgo/kv/kvtest" 8 | ) 9 | 10 | func TestLeveldb(t *testing.T) { 11 | kvtest.RunTestLocal(t, flat.UpgradeOpenPath(OpenPath), nil) 12 | } 13 | -------------------------------------------------------------------------------- /kv/flat/pebble/pebble.go: -------------------------------------------------------------------------------- 1 | //go:build !386 && !arm 2 | 3 | // Pebble doesn't support 32bit 4 | 5 | package pebble 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | 11 | "github.com/cockroachdb/pebble" 12 | 13 | "github.com/hidal-go/hidalgo/base" 14 | "github.com/hidal-go/hidalgo/kv/flat" 15 | ) 16 | 17 | const ( 18 | Name = "pebble" 19 | ) 20 | 21 | func init() { 22 | flat.Register(flat.Registration{ 23 | Registration: base.Registration{ 24 | Name: Name, Title: "Pebble", 25 | Local: true, 26 | }, 27 | OpenPath: OpenPath, 28 | }) 29 | } 30 | 31 | var _ flat.KV = (*DB)(nil) 32 | 33 | // OpenPathOptions is similar to OpenPath, but allow customizing Pebble options. 34 | func OpenPathOptions(path string, opts *pebble.Options) (*DB, error) { 35 | db, err := pebble.Open(path, opts) 36 | if err != nil { 37 | return nil, err 38 | } 39 | return &DB{db: db}, nil 40 | } 41 | 42 | func OpenPath(path string) (flat.KV, error) { 43 | db, err := OpenPathOptions(path, &pebble.Options{}) 44 | if err != nil { 45 | return nil, err 46 | } 47 | return db, nil 48 | } 49 | 50 | type DB struct { 51 | db *pebble.DB 52 | closed bool 53 | } 54 | 55 | func (db *DB) DB() *pebble.DB { 56 | return db.db 57 | } 58 | 59 | func (db *DB) Close() error { 60 | if db.closed { 61 | return nil 62 | } 63 | db.closed = true 64 | return db.db.Close() 65 | } 66 | 67 | func (db *DB) Tx(ctx context.Context, rw bool) (flat.Tx, error) { 68 | return &Tx{tx: db.db.NewIndexedBatch(), rw: rw}, nil 69 | } 70 | 71 | func (db *DB) View(ctx context.Context, fn func(tx flat.Tx) error) error { 72 | return flat.View(ctx, db, fn) 73 | } 74 | 75 | func (db *DB) Update(ctx context.Context, fn func(tx flat.Tx) error) error { 76 | return flat.Update(ctx, db, fn) 77 | } 78 | 79 | type Tx struct { 80 | tx *pebble.Batch 81 | rw bool 82 | } 83 | 84 | func (tx *Tx) Commit(ctx context.Context) error { 85 | if !tx.rw { 86 | return flat.ErrReadOnly 87 | } 88 | return tx.tx.Commit(pebble.Sync) 89 | } 90 | 91 | func (tx *Tx) Close() error { 92 | return tx.tx.Close() 93 | } 94 | 95 | func (tx *Tx) Get(ctx context.Context, key flat.Key) (flat.Value, error) { 96 | if len(key) == 0 { 97 | return nil, flat.ErrNotFound 98 | } 99 | val, closer, err := tx.tx.Get(key) 100 | if err == pebble.ErrNotFound { 101 | return nil, flat.ErrNotFound 102 | } else if err != nil { 103 | return nil, err 104 | } 105 | defer closer.Close() 106 | 107 | ret := make([]byte, len(val)) 108 | copy(ret, val) 109 | return ret, nil 110 | } 111 | 112 | func (tx *Tx) GetBatch(ctx context.Context, keys []flat.Key) ([]flat.Value, error) { 113 | return flat.GetBatch(ctx, tx, keys) 114 | } 115 | 116 | func (tx *Tx) Put(ctx context.Context, k flat.Key, v flat.Value) error { 117 | if !tx.rw { 118 | return flat.ErrReadOnly 119 | } 120 | return tx.tx.Set(k, v, pebble.Sync) 121 | } 122 | 123 | func (tx *Tx) Del(ctx context.Context, k flat.Key) error { 124 | if !tx.rw { 125 | return flat.ErrReadOnly 126 | } 127 | return tx.tx.Delete(k, pebble.Sync) 128 | } 129 | 130 | func (tx *Tx) Scan(ctx context.Context, opts ...flat.IteratorOption) flat.Iterator { 131 | pit := tx.tx.NewIter(nil) 132 | var it flat.Iterator = &Iterator{it: pit, first: true} 133 | it = flat.ApplyIteratorOptions(it, opts) 134 | return it 135 | } 136 | 137 | var ( 138 | _ flat.Seeker = &Iterator{} 139 | _ flat.PrefixIterator = &Iterator{} 140 | ) 141 | 142 | type Iterator struct { 143 | it *pebble.Iterator 144 | pref flat.Key 145 | first bool 146 | err error 147 | } 148 | 149 | func (it *Iterator) Reset() { 150 | it.first = true 151 | it.err = nil 152 | } 153 | 154 | func (it *Iterator) WithPrefix(pref flat.Key) flat.Iterator { 155 | it.Reset() 156 | it.pref = pref 157 | return it 158 | } 159 | 160 | func (it *Iterator) Seek(ctx context.Context, key flat.Key) bool { 161 | it.Reset() 162 | it.first = false 163 | it.it.SeekGE(key) 164 | return it.isValid() 165 | } 166 | 167 | func (it *Iterator) Next(ctx context.Context) bool { 168 | if it.first { 169 | it.first = false 170 | it.it.SeekGE(it.pref) 171 | } else { 172 | it.it.Next() 173 | } 174 | 175 | return it.isValid() 176 | } 177 | 178 | func (it *Iterator) Err() error { 179 | return it.err 180 | } 181 | 182 | func (it *Iterator) Close() error { 183 | it.it.Close() 184 | return it.Err() 185 | } 186 | 187 | func (it *Iterator) Key() flat.Key { 188 | return it.it.Key() 189 | } 190 | 191 | func (it *Iterator) Val() flat.Value { 192 | return it.it.Value() 193 | } 194 | 195 | func (it *Iterator) isValid() bool { 196 | if !it.it.Valid() { 197 | return false 198 | } 199 | 200 | if len(it.pref) != 0 && !bytes.HasPrefix(it.it.Key(), it.pref) { 201 | return false 202 | } 203 | 204 | return true 205 | } 206 | -------------------------------------------------------------------------------- /kv/flat/pebble/pebble_test.go: -------------------------------------------------------------------------------- 1 | //go:build !386 && !arm 2 | 3 | package pebble 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/hidal-go/hidalgo/kv/flat" 9 | "github.com/hidal-go/hidalgo/kv/kvtest" 10 | ) 11 | 12 | func TestPebble(t *testing.T) { 13 | kvtest.RunTestLocal(t, flat.UpgradeOpenPath(OpenPath), &kvtest.Options{ 14 | NoTx: true, 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /kv/flat/registry.go: -------------------------------------------------------------------------------- 1 | package flat 2 | 3 | import ( 4 | "sort" 5 | 6 | "github.com/hidal-go/hidalgo/base" 7 | "github.com/hidal-go/hidalgo/kv" 8 | ) 9 | 10 | // OpenPathFunc is a function for opening a database given a path. 11 | type OpenPathFunc func(path string) (KV, error) 12 | 13 | // Registration is an information about the database driver. 14 | type Registration struct { 15 | base.Registration 16 | OpenPath OpenPathFunc 17 | } 18 | 19 | var registry = make(map[string]Registration) 20 | 21 | // Register globally registers a database driver. 22 | func Register(reg Registration) { 23 | if reg.Name == "" { 24 | panic("name cannot be empty") 25 | } else if _, ok := registry[reg.Name]; ok { 26 | panic(base.ErrRegistered{Name: reg.Name}) 27 | } 28 | registry[reg.Name] = reg 29 | 30 | // Register as a hierarchical KV implementation 31 | reg.Name = "flat" + base.RegistrySep + reg.Name 32 | kv.Register(kv.Registration{ 33 | Registration: reg.Registration, 34 | OpenPath: func(path string) (kv.KV, error) { 35 | flat, err := reg.OpenPath(path) 36 | if err != nil { 37 | return nil, err 38 | } 39 | return Upgrade(flat), nil 40 | }, 41 | }) 42 | } 43 | 44 | // List enumerates all globally registered database drivers. 45 | func List() []Registration { 46 | out := make([]Registration, 0, len(registry)) 47 | for _, r := range registry { 48 | out = append(out, r) 49 | } 50 | sort.Slice(out, func(i, j int) bool { 51 | return out[i].Name < out[j].Name 52 | }) 53 | return out 54 | } 55 | 56 | // ByName returns a registered database driver by it's name. 57 | func ByName(name string) *Registration { 58 | r, ok := registry[name] 59 | if !ok { 60 | return nil 61 | } 62 | return &r 63 | } 64 | -------------------------------------------------------------------------------- /kv/flat/upgrade.go: -------------------------------------------------------------------------------- 1 | package flat 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/hidal-go/hidalgo/kv" 7 | ) 8 | 9 | var _ kv.KV = (*hieKV)(nil) 10 | 11 | const ( 12 | sep = '/' 13 | esc = '\\' 14 | ) 15 | 16 | // Upgrade upgrades flat KV to hierarchical KV. 17 | func Upgrade(flat KV) kv.KV { 18 | return &hieKV{flat: flat} 19 | } 20 | 21 | // UpgradeOpenPath automatically upgrades flat KV to hierarchical KV on open. 22 | func UpgradeOpenPath(open OpenPathFunc) kv.OpenPathFunc { 23 | return func(path string) (kv.KV, error) { 24 | flat, err := open(path) 25 | if err != nil { 26 | return nil, err 27 | } 28 | return Upgrade(flat), nil 29 | } 30 | } 31 | 32 | type hieKV struct { 33 | flat KV 34 | } 35 | 36 | // KeyEscape converts kv.Key to a flat Key. 37 | func KeyEscape(k kv.Key) Key { 38 | var k2 Key 39 | for i, s := range k { 40 | if i != 0 { 41 | k2 = append(k2, sep) 42 | } 43 | for _, p := range s { 44 | if p == esc || p == sep { 45 | k2 = append(k2, esc) 46 | } 47 | k2 = append(k2, p) 48 | } 49 | } 50 | return k2 51 | } 52 | 53 | // KeyUnescape converts flat Key into kv.Key. 54 | func KeyUnescape(k Key) kv.Key { 55 | var ( 56 | k2 kv.Key 57 | cur Key 58 | ) 59 | for i := 0; i < len(k); i++ { 60 | p := k[i] 61 | if p == esc { 62 | cur = append(cur, k[i+1]) 63 | i++ 64 | continue 65 | } else if p == sep { 66 | k2 = append(k2, cur) 67 | cur = nil 68 | continue 69 | } 70 | cur = append(cur, p) 71 | } 72 | if cur != nil { 73 | k2 = append(k2, cur) 74 | } 75 | return k2 76 | } 77 | 78 | func (hkv *hieKV) Close() error { 79 | return hkv.flat.Close() 80 | } 81 | 82 | func (hkv *hieKV) Tx(ctx context.Context, rw bool) (kv.Tx, error) { 83 | tx, err := hkv.flat.Tx(ctx, rw) 84 | if err != nil { 85 | return nil, err 86 | } 87 | return &flatTx{kv: hkv, tx: tx, rw: rw}, nil 88 | } 89 | 90 | func (hkv *hieKV) View(ctx context.Context, fn func(tx kv.Tx) error) error { 91 | return kv.View(ctx, hkv, fn) 92 | } 93 | 94 | func (hkv *hieKV) Update(ctx context.Context, fn func(tx kv.Tx) error) error { 95 | return kv.Update(ctx, hkv, fn) 96 | } 97 | 98 | type flatTx struct { 99 | kv *hieKV 100 | tx Tx 101 | rw bool 102 | } 103 | 104 | func (tx *flatTx) key(key kv.Key) Key { 105 | if len(key) == 0 { 106 | return nil 107 | } 108 | return KeyEscape(key) 109 | } 110 | 111 | func (tx *flatTx) Get(ctx context.Context, key kv.Key) (kv.Value, error) { 112 | return tx.tx.Get(ctx, tx.key(key)) 113 | } 114 | 115 | func (tx *flatTx) GetBatch(ctx context.Context, keys []kv.Key) ([]kv.Value, error) { 116 | ks := make([]Key, len(keys)) 117 | for i, k := range keys { 118 | ks[i] = tx.key(k) 119 | } 120 | return tx.tx.GetBatch(ctx, ks) 121 | } 122 | 123 | func (tx *flatTx) Commit(ctx context.Context) error { 124 | return tx.tx.Commit(ctx) 125 | } 126 | 127 | func (tx *flatTx) Close() error { 128 | return tx.tx.Close() 129 | } 130 | 131 | func (tx *flatTx) Put(ctx context.Context, k kv.Key, v kv.Value) error { 132 | if !tx.rw { 133 | return kv.ErrReadOnly 134 | } 135 | return tx.tx.Put(ctx, tx.key(k), v) 136 | } 137 | 138 | func (tx *flatTx) Del(ctx context.Context, k kv.Key) error { 139 | if !tx.rw { 140 | return kv.ErrReadOnly 141 | } 142 | return tx.tx.Del(ctx, tx.key(k)) 143 | } 144 | 145 | func (tx *flatTx) Scan(ctx context.Context, opts ...kv.IteratorOption) kv.Iterator { 146 | var ( 147 | native []IteratorOption 148 | fallback []kv.IteratorOption 149 | ) 150 | for _, opt := range opts { 151 | if v, ok := opt.(IteratorOption); ok { 152 | native = append(native, v) 153 | } else { 154 | fallback = append(fallback, opt) 155 | } 156 | } 157 | it := &prefIter{kv: tx.kv, Iterator: tx.tx.Scan(ctx, native...)} 158 | return kv.ApplyIteratorOptions(it, fallback) 159 | } 160 | 161 | type prefIter struct { 162 | kv *hieKV 163 | Iterator 164 | } 165 | 166 | func (it *prefIter) Val() kv.Value { 167 | return it.Iterator.Val() 168 | } 169 | 170 | func (it *prefIter) Key() kv.Key { 171 | return KeyUnescape(it.Iterator.Key()) 172 | } 173 | -------------------------------------------------------------------------------- /kv/flat/upgrade_test.go: -------------------------------------------------------------------------------- 1 | package flat 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/hidal-go/hidalgo/kv" 9 | ) 10 | 11 | func TestSepEscape(t *testing.T) { 12 | k := kv.Key{ 13 | []byte(`\/aa/b\b/c/d/\`), 14 | []byte(`/aa/b\b/c/d/`), 15 | } 16 | k2 := KeyUnescape(KeyEscape(k)) 17 | require.Equal(t, k, k2) 18 | } 19 | -------------------------------------------------------------------------------- /kv/helpers.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import "context" 4 | 5 | // Update is a helper to open a read-write transaction and update the database. 6 | // The update function may be called multiple times in case of conflicts with other writes. 7 | func Update(ctx context.Context, kv KV, update func(tx Tx) error) error { 8 | for { 9 | err := func() error { 10 | tx, err := kv.Tx(ctx, true) 11 | if err != nil { 12 | return err 13 | } 14 | defer tx.Close() 15 | err = update(tx) 16 | if err != nil { 17 | return err 18 | } 19 | return tx.Commit(ctx) 20 | }() 21 | if err == ErrConflict { 22 | continue 23 | } else if err != nil { 24 | return err 25 | } 26 | return nil 27 | } 28 | } 29 | 30 | // View is a helper to open a read-only transaction to read the database. 31 | func View(ctx context.Context, kv KV, view func(tx Tx) error) error { 32 | tx, err := kv.Tx(ctx, false) 33 | if err != nil { 34 | return err 35 | } 36 | defer tx.Close() 37 | err = view(tx) 38 | if err == nil { 39 | err = tx.Close() 40 | } 41 | return err 42 | } 43 | 44 | // Each is a helper to enumerate all key-value pairs with a specific prefix. 45 | // See Iterator for rules of using returned values. 46 | func Each(ctx context.Context, tx Tx, fnc func(k Key, v Value) error, opts ...IteratorOption) error { 47 | it := tx.Scan(ctx, opts...) 48 | defer it.Close() 49 | for it.Next(ctx) { 50 | if err := fnc(it.Key(), it.Val()); err != nil { 51 | return err 52 | } 53 | } 54 | return it.Err() 55 | } 56 | 57 | // CreateBucket is a helper to create buckets upfront without writing any key-value pairs to it. 58 | func CreateBucket(ctx context.Context, tx Tx, key Key) error { 59 | key = key.Clone() 60 | key = append(key, nil) 61 | return tx.Put(ctx, key, nil) 62 | } 63 | -------------------------------------------------------------------------------- /kv/iterator.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/hidal-go/hidalgo/base" 7 | ) 8 | 9 | // Iterator is an iterator over hierarchical key-value store. 10 | type Iterator interface { 11 | base.Iterator 12 | // Reset the iterator to the starting state. Closed iterator can not reset. 13 | Reset() 14 | // Key return current key. Returned value will become invalid on Next or Close. 15 | // Caller should not modify or store the value - use Clone. 16 | Key() Key 17 | // Val return current value. Returned value will become invalid on Next or Close. 18 | // Caller should not modify or store the value - use Clone. 19 | Val() Value 20 | } 21 | 22 | type Seeker interface { 23 | Iterator 24 | // Seek the iterator to a given key. If the key does not exist, the next key is used. 25 | // Function returns false if there is no key greater or equal to a given one. 26 | Seek(ctx context.Context, key Key) bool 27 | } 28 | 29 | // Seek the iterator to a given key. If the key does not exist, the next key is used. 30 | // Function returns false if there is no key greater or equal to a given one. 31 | func Seek(ctx context.Context, it Iterator, key Key) bool { 32 | if it, ok := it.(Seeker); ok { 33 | return it.Seek(ctx, key) 34 | } 35 | if len(key) == 0 { 36 | it.Reset() // seek to the beginning 37 | return it.Next(ctx) 38 | } 39 | // check where we currently are 40 | k := it.Key() 41 | if len(k) == 0 { 42 | // might either be the beginning, or the end, so restart to be sure 43 | it.Reset() 44 | } else { 45 | switch k.Compare(key) { 46 | case 0: 47 | return true // already there 48 | case -1: 49 | // can seek forward without resetting 50 | case +1: 51 | it.Reset() // too far, must restart 52 | } 53 | } 54 | for it.Next(ctx) { 55 | k = it.Key() 56 | if k.Compare(key) >= 0 { 57 | return true 58 | } 59 | } 60 | return false 61 | } 62 | 63 | // ApplyIteratorOptions applies all iterator options. 64 | func ApplyIteratorOptions(it Iterator, opts []IteratorOption) Iterator { 65 | for _, opt := range opts { 66 | it = opt.ApplyKV(it) 67 | } 68 | return it 69 | } 70 | 71 | // IteratorOption is an additional option that affects iterator behaviour. 72 | // 73 | // Implementations in generic KV package should assert for an optimized version of option (via interface assertion), 74 | // and fallback to generic implementation if the store doesn't support this option natively. 75 | type IteratorOption interface { 76 | // ApplyKV applies option to the KV iterator. Implementation may wrap or replace the iterator. 77 | ApplyKV(it Iterator) Iterator 78 | } 79 | 80 | // IteratorOptionFunc is a function type that implements IteratorOption. 81 | type IteratorOptionFunc func(it Iterator) Iterator 82 | 83 | func (opt IteratorOptionFunc) Apply(it Iterator) Iterator { 84 | return opt(it) 85 | } 86 | 87 | // PrefixIterator is an Iterator optimization to support WithPrefix option. 88 | type PrefixIterator interface { 89 | Iterator 90 | // WithPrefix implements WithPrefix iterator option. 91 | // Current iterator will be replaced with a new one and must not be used after this call. 92 | WithPrefix(pref Key) Iterator 93 | } 94 | -------------------------------------------------------------------------------- /kv/kv.go: -------------------------------------------------------------------------------- 1 | // Package kv provides an abstraction over hierarchical key-value stores. 2 | package kv 3 | 4 | import ( 5 | "bytes" 6 | "context" 7 | "errors" 8 | 9 | "github.com/hidal-go/hidalgo/base" 10 | ) 11 | 12 | var ( 13 | // ErrNotFound is returned then a key was not found in the database. 14 | ErrNotFound = errors.New("kv: not found") 15 | // ErrReadOnly is returned when write operation is performed on read-only database or transaction. 16 | ErrReadOnly = errors.New("kv: read only") 17 | // ErrConflict is returned when write operation performed be current transaction cannot be committed 18 | // because of another concurrent write. Caller must restart the transaction. 19 | ErrConflict = errors.New("kv: read only") 20 | ) 21 | 22 | // KV is an interface for hierarchical key-value databases. 23 | type KV interface { 24 | base.DB 25 | Tx(ctx context.Context, rw bool) (Tx, error) 26 | View(ctx context.Context, fn func(tx Tx) error) error 27 | Update(ctx context.Context, fn func(tx Tx) error) error 28 | } 29 | 30 | // Key is a hierarchical binary key used in a database. 31 | type Key [][]byte 32 | 33 | // SKey is a helper for making string keys. 34 | func SKey(parts ...string) Key { 35 | k := make(Key, 0, len(parts)) 36 | for _, s := range parts { 37 | k = append(k, []byte(s)) 38 | } 39 | return k 40 | } 41 | 42 | // Compare return 0 when keys are equal, -1 when k < k2 and +1 when k > k2. 43 | func (k Key) Compare(k2 Key) int { 44 | for i, s := range k { 45 | if i >= len(k2) { 46 | return +1 47 | } 48 | if d := bytes.Compare(s, k2[i]); d != 0 { 49 | return d 50 | } 51 | } 52 | if len(k) < len(k2) { 53 | return -1 54 | } 55 | return 0 56 | } 57 | 58 | // HasPrefix checks if a key has a given prefix. 59 | func (k Key) HasPrefix(pref Key) bool { 60 | if len(pref) == 0 { 61 | return true 62 | } else if len(k) < len(pref) { 63 | return false 64 | } 65 | for i, p := range pref { 66 | s := k[i] 67 | if i == len(pref)-1 { 68 | if !bytes.HasPrefix(s, p) { 69 | return false 70 | } 71 | } else { 72 | if !bytes.Equal(s, p) { 73 | return false 74 | } 75 | } 76 | } 77 | return true 78 | } 79 | 80 | // Append key parts and return a new value. 81 | // Value is not a deep copy, use Clone for this. 82 | func (k Key) Append(parts Key) Key { 83 | if k == nil && parts == nil { 84 | return nil 85 | } 86 | k2 := make(Key, len(k)+len(parts)) 87 | i := copy(k2, k) 88 | copy(k2[i:], parts) 89 | return k2 90 | } 91 | 92 | // AppendBytes is the same like Append, but accepts bytes slices. 93 | func (k Key) AppendBytes(parts ...[]byte) Key { 94 | return k.Append(Key(parts)) 95 | } 96 | 97 | // Clone returns a copy of the key. 98 | func (k Key) Clone() Key { 99 | if k == nil { 100 | return nil 101 | } 102 | p := make(Key, len(k)) 103 | for i, sk := range k { 104 | p[i] = make([]byte, len(sk)) 105 | copy(p[i], sk) 106 | } 107 | return p 108 | } 109 | 110 | // ByKey sorts keys in ascending order. 111 | type ByKey []Key 112 | 113 | func (s ByKey) Len() int { 114 | return len(s) 115 | } 116 | 117 | func (s ByKey) Less(i, j int) bool { 118 | return s[i].Compare(s[j]) < 0 119 | } 120 | 121 | func (s ByKey) Swap(i, j int) { 122 | s[i], s[j] = s[j], s[i] 123 | } 124 | 125 | // Value is a binary value stored in a database. 126 | type Value []byte 127 | 128 | // Clone returns a copy of the value. 129 | func (v Value) Clone() Value { 130 | if v == nil { 131 | return nil 132 | } 133 | p := make(Value, len(v)) 134 | copy(p, v) 135 | return p 136 | } 137 | 138 | // Pair is a key-value pair. 139 | type Pair struct { 140 | Key Key 141 | Val Value 142 | } 143 | 144 | type Getter interface { 145 | // Get fetches a value for a single key from the database. 146 | // It return ErrNotFound if key does not exists. 147 | Get(ctx context.Context, key Key) (Value, error) 148 | } 149 | 150 | // Tx is a transaction over hierarchical key-value store. 151 | type Tx interface { 152 | base.Tx 153 | Getter 154 | // GetBatch fetches values for multiple keys from the database. 155 | // Nil element in the slice indicates that key does not exists. 156 | GetBatch(ctx context.Context, keys []Key) ([]Value, error) 157 | // Put writes a key-value pair to the database. 158 | // New value will immediately be visible by Get on the same Tx, 159 | // but implementation might buffer the write until transaction is committed. 160 | Put(ctx context.Context, k Key, v Value) error 161 | // Del removes the key from the database. See Put for consistency guaranties. 162 | Del(ctx context.Context, k Key) error 163 | // Scan starts iteration over key-value pairs. Returned results are affected by IteratorOption. 164 | Scan(ctx context.Context, opts ...IteratorOption) Iterator 165 | } 166 | 167 | // GetBatch is an implementation of Tx.GetBatch for databases that has no native implementation for it. 168 | func GetBatch(ctx context.Context, tx Getter, keys []Key) ([]Value, error) { 169 | vals := make([]Value, len(keys)) 170 | var err error 171 | for i, k := range keys { 172 | vals[i], err = tx.Get(ctx, k) 173 | if err == ErrNotFound { 174 | vals[i] = nil 175 | } else if err != nil { 176 | return nil, err 177 | } 178 | } 179 | return vals, nil 180 | } 181 | -------------------------------------------------------------------------------- /kv/kv_test.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | var keyCompareCases = []struct { 10 | k1, k2 Key 11 | exp int 12 | }{ 13 | { 14 | k1: SKey("a"), 15 | k2: SKey("a"), 16 | exp: 0, 17 | }, 18 | { 19 | k1: SKey("a"), 20 | k2: SKey("b"), 21 | exp: -1, 22 | }, 23 | { 24 | k1: SKey("a"), 25 | k2: SKey("a", "b"), 26 | exp: -1, 27 | }, 28 | { 29 | k1: SKey("a", "b"), 30 | k2: SKey("a", "a"), 31 | exp: +1, 32 | }, 33 | { 34 | k1: nil, 35 | k2: SKey("a", "b"), 36 | exp: -1, 37 | }, 38 | { 39 | k1: SKey("ab"), 40 | k2: SKey("a", "b"), 41 | exp: +1, 42 | }, 43 | } 44 | 45 | func TestKeyCompare(t *testing.T) { 46 | for _, c := range keyCompareCases { 47 | t.Run("", func(t *testing.T) { 48 | d := c.k1.Compare(c.k2) 49 | require.Equal(t, c.exp, d) 50 | if d != 0 { 51 | require.Equal(t, -c.exp, c.k2.Compare(c.k1)) 52 | } 53 | }) 54 | } 55 | } 56 | 57 | var keyHasPrefixCases = []struct { 58 | key, pref Key 59 | exp bool 60 | }{ 61 | { 62 | key: SKey("a"), 63 | pref: SKey("a"), 64 | exp: true, 65 | }, 66 | { 67 | key: SKey("a"), 68 | pref: SKey("b"), 69 | exp: false, 70 | }, 71 | { 72 | key: SKey("a"), 73 | pref: SKey("a", "b"), 74 | exp: false, 75 | }, 76 | { 77 | key: SKey("a", "b"), 78 | pref: SKey("a"), 79 | exp: true, 80 | }, 81 | { 82 | key: SKey("a", "b"), 83 | pref: SKey("a", "a"), 84 | exp: false, 85 | }, 86 | { 87 | key: nil, 88 | pref: SKey("a", "b"), 89 | exp: false, 90 | }, 91 | { 92 | key: SKey("a", "b"), 93 | pref: nil, 94 | exp: true, 95 | }, 96 | { 97 | key: SKey("ab"), 98 | pref: SKey("a", "b"), 99 | exp: false, 100 | }, 101 | { 102 | key: SKey("a", "b"), 103 | pref: SKey("ab"), 104 | exp: false, 105 | }, 106 | } 107 | 108 | func TestKeyHasPrefix(t *testing.T) { 109 | for _, c := range keyHasPrefixCases { 110 | t.Run("", func(t *testing.T) { 111 | d := c.key.HasPrefix(c.pref) 112 | require.Equal(t, c.exp, d) 113 | }) 114 | } 115 | } 116 | 117 | func TestKeyAppend(t *testing.T) { 118 | k := SKey("a", "b", "c") 119 | k = k.Append(SKey("d")) 120 | require.Equal(t, SKey("a", "b", "c", "d"), k) 121 | k = SKey("a", "b", "c") 122 | k = k.AppendBytes([]byte("d")) 123 | require.Equal(t, SKey("a", "b", "c", "d"), k) 124 | } 125 | -------------------------------------------------------------------------------- /kv/kvdebug/kvdebug.go: -------------------------------------------------------------------------------- 1 | package kvdebug 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "sync/atomic" 9 | 10 | "github.com/hidal-go/hidalgo/kv" 11 | ) 12 | 13 | var _ kv.KV = (*KV)(nil) 14 | 15 | func New(kv kv.KV) *KV { 16 | return &KV{KV: kv} 17 | } 18 | 19 | type Stats struct { 20 | Errs int64 21 | Tx struct { 22 | RO int64 23 | RW int64 24 | } 25 | Get struct { 26 | N int64 27 | Batch int64 28 | Miss int64 29 | } 30 | Put struct { 31 | N int64 32 | } 33 | Del struct { 34 | N int64 35 | } 36 | Iter struct { 37 | N int64 38 | Next int64 39 | K, V int64 40 | } 41 | } 42 | 43 | type KV struct { 44 | stats Stats 45 | running struct { 46 | txRO int64 47 | txRW int64 48 | iter int64 49 | } 50 | log bool 51 | 52 | KV kv.KV 53 | } 54 | 55 | func (d *KV) logging() bool { 56 | return d.log 57 | } 58 | 59 | func (d *KV) Log(v bool) { 60 | d.log = v 61 | } 62 | 63 | func (d *KV) Stats() Stats { 64 | return d.stats 65 | } 66 | 67 | func (d *KV) Close() error { 68 | err := d.KV.Close() 69 | if err != nil { 70 | atomic.AddInt64(&d.stats.Errs, 1) 71 | } 72 | r := atomic.LoadInt64(&d.running.txRO) 73 | w := atomic.LoadInt64(&d.running.txRW) 74 | s := atomic.LoadInt64(&d.running.iter) 75 | if r+w+s != 0 { 76 | panic(fmt.Errorf("resources leak: iter: %d, ro: %d, rw: %d", s, r, w)) 77 | } 78 | return err 79 | } 80 | 81 | func (d *KV) Tx(ctx context.Context, rw bool) (kv.Tx, error) { 82 | tx, err := d.KV.Tx(ctx, rw) 83 | if err != nil { 84 | if tx != nil { 85 | panic("tx should be nil on error") 86 | } 87 | atomic.AddInt64(&d.stats.Errs, 1) 88 | return nil, err 89 | } 90 | if rw { 91 | atomic.AddInt64(&d.stats.Tx.RW, 1) 92 | atomic.AddInt64(&d.running.txRW, 1) 93 | } else { 94 | atomic.AddInt64(&d.stats.Tx.RO, 1) 95 | atomic.AddInt64(&d.running.txRO, 1) 96 | } 97 | return &kvTX{kv: d, tx: tx, rw: rw}, nil 98 | } 99 | 100 | func (d *KV) View(ctx context.Context, fn func(tx kv.Tx) error) error { 101 | return kv.View(ctx, d, fn) 102 | } 103 | 104 | func (d *KV) Update(ctx context.Context, fn func(tx kv.Tx) error) error { 105 | return kv.Update(ctx, d, fn) 106 | } 107 | 108 | type kvTX struct { 109 | kv *KV 110 | tx kv.Tx 111 | err error 112 | rw bool 113 | } 114 | 115 | func (tx *kvTX) done(err error) { 116 | tx.err = err 117 | tx.tx = nil 118 | 119 | d := tx.kv 120 | if err != nil { 121 | atomic.AddInt64(&d.stats.Errs, 1) 122 | } 123 | if tx.rw { 124 | atomic.AddInt64(&d.running.txRW, -1) 125 | } else { 126 | atomic.AddInt64(&d.running.txRO, -1) 127 | } 128 | } 129 | 130 | func (tx *kvTX) Commit(ctx context.Context) error { 131 | if tx.tx == nil { 132 | return tx.err 133 | } 134 | err := tx.tx.Commit(ctx) 135 | tx.done(err) 136 | return err 137 | } 138 | 139 | func (tx *kvTX) Close() error { 140 | if tx.tx == nil { 141 | return tx.err 142 | } 143 | err := tx.tx.Close() 144 | tx.done(err) 145 | return err 146 | } 147 | 148 | func (tx *kvTX) Get(ctx context.Context, k kv.Key) (kv.Value, error) { 149 | v, err := tx.tx.Get(ctx, k) 150 | d := tx.kv 151 | atomic.AddInt64(&d.stats.Get.N, 1) 152 | if err == kv.ErrNotFound { 153 | atomic.AddInt64(&d.stats.Get.Miss, 1) 154 | } else if err != nil { 155 | atomic.AddInt64(&d.stats.Errs, 1) 156 | } 157 | if d.logging() { 158 | log.Printf("get: %q = %q (%v)", k, v, err) 159 | } 160 | return v, err 161 | } 162 | 163 | func (tx *kvTX) GetBatch(ctx context.Context, keys []kv.Key) ([]kv.Value, error) { 164 | vals, err := tx.tx.GetBatch(ctx, keys) 165 | d := tx.kv 166 | atomic.AddInt64(&d.stats.Get.Batch, int64(len(keys))) 167 | if err != nil { 168 | atomic.AddInt64(&d.stats.Errs, 1) 169 | } 170 | for _, v := range vals { 171 | if v == nil { 172 | atomic.AddInt64(&d.stats.Get.Miss, 1) 173 | } 174 | } 175 | if d.logging() { 176 | log.Printf("get batch: %d (%v)", len(keys), err) 177 | for i := range vals { 178 | log.Printf("get: %q = %q", keys[i], vals[i]) 179 | } 180 | } 181 | return vals, err 182 | } 183 | 184 | func (tx *kvTX) Put(ctx context.Context, k kv.Key, v kv.Value) error { 185 | if !tx.rw { 186 | return errors.New("read-only transaction") 187 | } 188 | err := tx.tx.Put(ctx, k, v) 189 | d := tx.kv 190 | atomic.AddInt64(&d.stats.Put.N, 1) 191 | if err != nil { 192 | atomic.AddInt64(&d.stats.Errs, 1) 193 | } 194 | if d.logging() { 195 | log.Printf("put: %q = %q (%v)", k, v, err) 196 | } 197 | return err 198 | } 199 | 200 | func (tx *kvTX) Del(ctx context.Context, k kv.Key) error { 201 | if !tx.rw { 202 | panic("del in RO transaction") 203 | } 204 | err := tx.tx.Del(ctx, k) 205 | d := tx.kv 206 | atomic.AddInt64(&d.stats.Del.N, 1) 207 | if err != nil { 208 | atomic.AddInt64(&d.stats.Errs, 1) 209 | } 210 | if d.logging() { 211 | log.Printf("del: %q (%v)", k, err) 212 | } 213 | return err 214 | } 215 | 216 | func (tx *kvTX) Scan(ctx context.Context, opts ...kv.IteratorOption) kv.Iterator { 217 | d := tx.kv 218 | atomic.AddInt64(&d.running.iter, 1) 219 | atomic.AddInt64(&d.stats.Iter.N, 1) 220 | if d.logging() { 221 | log.Printf("scan: %+v", opts) 222 | } 223 | return &kvIter{kv: tx.kv, it: tx.tx.Scan(ctx, opts...)} 224 | } 225 | 226 | type kvIter struct { 227 | kv *KV 228 | it kv.Iterator 229 | err error 230 | } 231 | 232 | func (it *kvIter) Reset() { 233 | d := it.kv 234 | it.it.Reset() 235 | it.err = nil 236 | if d.logging() { 237 | log.Printf("reset") 238 | } 239 | } 240 | 241 | func (it *kvIter) Next(ctx context.Context) bool { 242 | d := it.kv 243 | if !it.it.Next(ctx) { 244 | if d.logging() { 245 | log.Printf("scan: %v", false) 246 | } 247 | return false 248 | } 249 | atomic.AddInt64(&d.stats.Iter.Next, 1) 250 | if d.logging() { 251 | log.Printf("scan: %q = %q", it.it.Key(), it.it.Val()) 252 | } 253 | return true 254 | } 255 | 256 | func (it *kvIter) Err() error { 257 | return it.it.Err() 258 | } 259 | 260 | func (it *kvIter) Close() error { 261 | if it.it == nil { 262 | return it.err 263 | } 264 | err := it.it.Close() 265 | it.err = err 266 | it.it = nil 267 | 268 | d := it.kv 269 | if err != nil { 270 | atomic.AddInt64(&d.stats.Errs, 1) 271 | } 272 | atomic.AddInt64(&d.running.iter, -1) 273 | return err 274 | } 275 | 276 | func (it *kvIter) Key() kv.Key { 277 | d := it.kv 278 | atomic.AddInt64(&d.stats.Iter.K, 1) 279 | return it.it.Key() 280 | } 281 | 282 | func (it *kvIter) Val() kv.Value { 283 | d := it.kv 284 | atomic.AddInt64(&d.stats.Iter.V, 1) 285 | return it.it.Val() 286 | } 287 | -------------------------------------------------------------------------------- /kv/kvtest/helpers.go: -------------------------------------------------------------------------------- 1 | package kvtest 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/hidal-go/hidalgo/kv" 10 | ) 11 | 12 | func NewTest(t testing.TB, db kv.KV) *Test { 13 | return &Test{t: t, db: db} 14 | } 15 | 16 | type Test struct { 17 | t testing.TB 18 | db kv.KV 19 | } 20 | 21 | func (t Test) Get(key kv.Key) (kv.Value, error) { 22 | ctx := context.Background() 23 | tx, err := t.db.Tx(ctx, false) 24 | require.NoError(t.t, err) 25 | defer tx.Close() 26 | return tx.Get(ctx, key) 27 | } 28 | 29 | func (t Test) NotExists(k kv.Key) { 30 | v, err := t.Get(k) 31 | require.Equal(t.t, kv.ErrNotFound, err) 32 | require.Equal(t.t, kv.Value(nil), v) 33 | } 34 | 35 | func (t Test) Expect(k kv.Key, exp kv.Value) { 36 | v, err := t.Get(k) 37 | require.NoError(t.t, err) 38 | require.Equal(t.t, exp, v) 39 | } 40 | 41 | func (t Test) Put(key kv.Key, val kv.Value) { 42 | ctx := context.Background() 43 | tx, err := t.db.Tx(ctx, true) 44 | require.NoError(t.t, err) 45 | defer tx.Close() 46 | err = tx.Put(ctx, key, val) 47 | require.NoError(t.t, err) 48 | 49 | got, err := tx.Get(ctx, key) 50 | require.NoError(t.t, err) 51 | require.Equal(t.t, val, got) 52 | err = tx.Commit(ctx) 53 | require.NoError(t.t, err) 54 | } 55 | 56 | func (t Test) Del(key kv.Key) { 57 | ctx := context.Background() 58 | tx, err := t.db.Tx(ctx, true) 59 | require.NoError(t.t, err) 60 | defer tx.Close() 61 | 62 | err = tx.Del(ctx, key) 63 | require.NoError(t.t, err) 64 | 65 | got, err := tx.Get(ctx, key) 66 | require.Equal(t.t, kv.ErrNotFound, err) 67 | require.Equal(t.t, kv.Value(nil), got) 68 | 69 | err = tx.Commit(ctx) 70 | require.NoError(t.t, err) 71 | } 72 | 73 | func (t Test) ExpectIt(it kv.Iterator, exp []kv.Pair) { 74 | if len(exp) == 0 { 75 | exp = nil 76 | } 77 | ctx := context.Background() 78 | var got []kv.Pair 79 | for it.Next(ctx) { 80 | got = append(got, kv.Pair{ 81 | Key: it.Key().Clone(), 82 | Val: it.Val().Clone(), 83 | }) 84 | } 85 | require.NoError(t.t, it.Err()) 86 | require.Equal(t.t, exp, got) 87 | } 88 | 89 | func (t Test) Scan(exp []kv.Pair, opts ...kv.IteratorOption) { 90 | ctx := context.Background() 91 | tx, err := t.db.Tx(ctx, false) 92 | require.NoError(t.t, err) 93 | defer tx.Close() 94 | 95 | it := tx.Scan(ctx, opts...) 96 | defer it.Close() 97 | require.NoError(t.t, it.Err()) 98 | 99 | t.ExpectIt(it, exp) 100 | } 101 | 102 | func (t Test) ScanReset(exp []kv.Pair, opts ...kv.IteratorOption) { 103 | ctx := context.Background() 104 | tx, err := t.db.Tx(ctx, false) 105 | require.NoError(t.t, err) 106 | defer tx.Close() 107 | 108 | it := tx.Scan(ctx, opts...) 109 | defer it.Close() 110 | require.NoError(t.t, it.Err()) 111 | 112 | t.ExpectIt(it, exp) 113 | it.Reset() 114 | t.ExpectIt(it, exp) 115 | } 116 | -------------------------------------------------------------------------------- /kv/kvtest/kvtest.go: -------------------------------------------------------------------------------- 1 | package kvtest 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "os" 7 | "strconv" 8 | "sync" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/require" 12 | 13 | "github.com/hidal-go/hidalgo/kv" 14 | "github.com/hidal-go/hidalgo/kv/options" 15 | ) 16 | 17 | // Func is a constructor for database implementations. 18 | // It returns an empty database and a function to destroy it. 19 | type Func func(t testing.TB) kv.KV 20 | 21 | type Options struct { 22 | NoLocks bool // not safe for concurrent writes 23 | NoTx bool // implementation doesn't support proper transactions 24 | } 25 | 26 | // RunTest runs all tests for key-value implementations. 27 | func RunTest(t *testing.T, fnc Func, opts *Options) { 28 | if opts == nil { 29 | opts = &Options{} 30 | } 31 | for _, c := range testList { 32 | t.Run(c.name, func(t *testing.T) { 33 | if c.concurrent && opts.NoLocks { 34 | t.Skip("implementation doesn't support concurrent writes") 35 | } 36 | if c.txOnly && opts.NoTx { 37 | t.Skip("implementation doesn't support transactions") 38 | } 39 | db := fnc(t) 40 | c.test(t, db) 41 | }) 42 | } 43 | } 44 | 45 | // RunTestLocal is a wrapper for RunTest that automatically creates a temporary directory and opens a database. 46 | func RunTestLocal(t *testing.T, open kv.OpenPathFunc, opts *Options) { 47 | if opts == nil { 48 | opts = &Options{} 49 | } 50 | RunTest(t, func(t testing.TB) kv.KV { 51 | dir, err := ioutil.TempDir("", "dal-kv-") 52 | require.NoError(t, err) 53 | t.Cleanup(func() { 54 | _ = os.RemoveAll(dir) 55 | }) 56 | 57 | db, err := open(dir) 58 | if err != nil { 59 | require.NoError(t, err) 60 | } 61 | t.Cleanup(func() { 62 | db.Close() 63 | db.Close() // test double close 64 | }) 65 | return db 66 | }, opts) 67 | } 68 | 69 | var testList = []struct { 70 | name string 71 | test func(t testing.TB, db kv.KV) 72 | concurrent bool // requires concurrent safety 73 | txOnly bool // requires transactions 74 | }{ 75 | {name: "basic", test: basic}, 76 | {name: "ro", test: readonly}, 77 | {name: "seek", test: seek}, 78 | {name: "increment", test: increment, txOnly: true, concurrent: true}, 79 | } 80 | 81 | func basic(t testing.TB, db kv.KV) { 82 | td := NewTest(t, db) 83 | 84 | keys := []kv.Key{ 85 | {[]byte("a")}, 86 | {[]byte("b"), []byte("a")}, 87 | {[]byte("b"), []byte("a1")}, 88 | {[]byte("b"), []byte("a2")}, 89 | {[]byte("b"), []byte("b")}, 90 | {[]byte("c")}, 91 | } 92 | 93 | td.NotExists(nil) 94 | for _, k := range keys { 95 | td.NotExists(k) 96 | } 97 | 98 | var all []kv.Pair 99 | for i, k := range keys { 100 | v := kv.Value(strconv.Itoa(i)) 101 | td.Put(k, v) 102 | td.Expect(k, v) 103 | all = append(all, kv.Pair{Key: k, Val: v}) 104 | } 105 | 106 | td.ScanReset(all) 107 | td.ScanReset(all[:1], options.WithPrefixKV(keys[0])) 108 | td.ScanReset(all[len(all)-1:], options.WithPrefixKV(keys[len(keys)-1])) 109 | td.ScanReset(all[1:len(all)-1], options.WithPrefixKV(keys[1][:1])) 110 | td.ScanReset(all[1:4], options.WithPrefixKV(kv.Key{keys[1][0], keys[1][1][:1]})) 111 | 112 | for _, k := range keys { 113 | td.Del(k) 114 | } 115 | for _, k := range keys { 116 | td.NotExists(k) 117 | } 118 | } 119 | 120 | func readonly(t testing.TB, db kv.KV) { 121 | td := NewTest(t, db) 122 | 123 | key := kv.Key{[]byte("a")} 124 | val := []byte("v") 125 | td.Put(key, val) 126 | 127 | nokey := kv.Key{[]byte("b")} 128 | 129 | ctx := context.Background() 130 | tx, err := db.Tx(ctx, false) 131 | require.NoError(t, err) 132 | defer tx.Close() 133 | 134 | // writing anything on read-only tx must fail 135 | err = tx.Put(ctx, key, val) 136 | require.Equal(t, kv.ErrReadOnly, err) 137 | err = tx.Put(ctx, nokey, val) 138 | require.Equal(t, kv.ErrReadOnly, err) 139 | 140 | // deleting records on read-only tx must fail 141 | err = tx.Del(ctx, key) 142 | require.Equal(t, kv.ErrReadOnly, err) 143 | 144 | // deleting non-existed record on read-only tx must still fail 145 | err = tx.Del(ctx, nokey) 146 | require.Equal(t, kv.ErrReadOnly, err) 147 | } 148 | 149 | func seek(t testing.TB, db kv.KV) { 150 | td := NewTest(t, db) 151 | 152 | keys := []kv.Key{ 153 | {[]byte("a")}, 154 | {[]byte("b"), []byte("a")}, 155 | {[]byte("b"), []byte("a1")}, 156 | {[]byte("b"), []byte("a2")}, 157 | {[]byte("b"), []byte("b")}, 158 | {[]byte("c")}, 159 | } 160 | 161 | var all []kv.Pair 162 | for i, k := range keys { 163 | v := kv.Value(strconv.Itoa(i)) 164 | td.Put(k, v) 165 | all = append(all, kv.Pair{Key: k, Val: v}) 166 | } 167 | 168 | ctx := context.Background() 169 | tx, err := db.Tx(ctx, false) 170 | require.NoError(t, err) 171 | defer tx.Close() 172 | 173 | // start an iterator and check if it can reset 174 | // this is the basic requirement for generic seek to work 175 | it := tx.Scan(ctx) 176 | defer it.Close() 177 | td.ExpectIt(it, all) 178 | it.Reset() 179 | td.ExpectIt(it, all) 180 | 181 | // seek to each key, current value must match corresponding element, and iterating further must return everything else 182 | for i, p := range all { 183 | ok := kv.Seek(ctx, it, p.Key) 184 | require.True(t, ok) 185 | require.Equal(t, p.Key, it.Key()) 186 | require.Equal(t, p.Val, it.Val()) 187 | td.ExpectIt(it, all[i+1:]) 188 | } 189 | 190 | // seek to element, then seek to the next key (test offsets 1 and 2) 191 | for _, off := range []int{1, 2} { 192 | for i := range all[:len(all)-off] { 193 | ok := kv.Seek(ctx, it, all[i].Key) 194 | require.True(t, ok) 195 | require.Equal(t, all[i].Key, it.Key()) 196 | require.Equal(t, all[i].Val, it.Val()) 197 | 198 | ok = kv.Seek(ctx, it, all[i+off].Key) 199 | require.True(t, ok) 200 | require.Equal(t, all[i+off].Key, it.Key()) 201 | require.Equal(t, all[i+off].Val, it.Val()) 202 | 203 | td.ExpectIt(it, all[i+off+1:]) 204 | } 205 | } 206 | 207 | // same, but seek backward instead 208 | for _, off := range []int{1, 2} { 209 | for i := range all { 210 | if i < off { 211 | continue 212 | } 213 | ok := kv.Seek(ctx, it, all[i].Key) 214 | require.True(t, ok) 215 | require.Equal(t, all[i].Key, it.Key()) 216 | require.Equal(t, all[i].Val, it.Val()) 217 | 218 | ok = kv.Seek(ctx, it, all[i-off].Key) 219 | require.True(t, ok) 220 | require.Equal(t, all[i-off].Key, it.Key()) 221 | require.Equal(t, all[i-off].Val, it.Val()) 222 | 223 | td.ExpectIt(it, all[i-off+1:]) 224 | } 225 | } 226 | // regular reset still works 227 | it.Reset() 228 | td.ExpectIt(it, all) 229 | } 230 | 231 | func increment(t testing.TB, db kv.KV) { 232 | td := NewTest(t, db) 233 | 234 | key := kv.Key{[]byte("a")} 235 | td.Put(key, []byte("0")) 236 | 237 | const n = 10 238 | ready := make(chan struct{}) 239 | errc := make(chan error, n) 240 | var wg sync.WaitGroup 241 | for i := 0; i < n; i++ { 242 | wg.Add(1) 243 | go func() { 244 | defer wg.Done() 245 | ctx := context.Background() 246 | <-ready 247 | err := kv.Update(ctx, db, func(tx kv.Tx) error { 248 | val, err := tx.Get(ctx, key) 249 | if err != nil { 250 | return err 251 | } 252 | v, err := strconv.Atoi(string(val)) 253 | if err != nil { 254 | return err 255 | } 256 | v++ 257 | val = []byte(strconv.Itoa(v)) 258 | return tx.Put(ctx, key, val) 259 | }) 260 | if err != nil { 261 | errc <- err 262 | } 263 | }() 264 | } 265 | close(ready) 266 | wg.Wait() 267 | select { 268 | case err := <-errc: 269 | require.NoError(t, err) 270 | default: 271 | } 272 | td.Expect(key, []byte("10")) 273 | } 274 | -------------------------------------------------------------------------------- /kv/options/options.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import ( 4 | "github.com/hidal-go/hidalgo/kv" 5 | "github.com/hidal-go/hidalgo/kv/flat" 6 | ) 7 | 8 | var ( 9 | _ kv.IteratorOption = (IteratorOption)(nil) 10 | _ flat.IteratorOption = (IteratorOption)(nil) 11 | ) 12 | 13 | // IteratorOption is an additional option that affects iterator behaviour. 14 | // 15 | // Implementations in generic KV package should assert for an optimized version of option (via interface assertion), 16 | // and fallback to generic implementation if the store doesn't support this option natively. 17 | type IteratorOption interface { 18 | // ApplyKV applies option to the KV iterator. Implementation may wrap or replace the iterator. 19 | ApplyKV(it kv.Iterator) kv.Iterator 20 | // ApplyFlat option to the flat KV iterator. Implementation may wrap or replace the iterator. 21 | ApplyFlat(it flat.Iterator) flat.Iterator 22 | } 23 | -------------------------------------------------------------------------------- /kv/options/prefix_flat.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | 7 | "github.com/hidal-go/hidalgo/kv" 8 | "github.com/hidal-go/hidalgo/kv/flat" 9 | ) 10 | 11 | // WithPrefixFlat returns IteratorOption that limits scanned key to a given binary prefix. 12 | // Store implementations can optimize this by implementing flat.PrefixIterator. 13 | func WithPrefixFlat(pref flat.Key) IteratorOption { 14 | return PrefixFlat{Pref: pref} 15 | } 16 | 17 | // PrefixFlat implements IteratorOption. See WithPrefixFlat. 18 | type PrefixFlat struct { 19 | Pref flat.Key 20 | } 21 | 22 | func (opt PrefixFlat) ApplyFlat(it flat.Iterator) flat.Iterator { 23 | if it, ok := it.(flat.PrefixIterator); ok { 24 | return it.WithPrefix(opt.Pref) 25 | } 26 | return &prefixIteratorFlat{base: it, pref: opt.Pref} 27 | } 28 | 29 | func (opt PrefixFlat) ApplyKV(it kv.Iterator) kv.Iterator { 30 | pref := flat.KeyUnescape(opt.Pref) 31 | if it, ok := it.(kv.PrefixIterator); ok { 32 | return it.WithPrefix(pref) 33 | } 34 | return &prefixIteratorKV{base: it, pref: pref} 35 | } 36 | 37 | var _ flat.PrefixIterator = &prefixIteratorFlat{} 38 | 39 | type prefixIteratorFlat struct { 40 | base flat.Iterator 41 | pref flat.Key 42 | seek bool 43 | done bool 44 | } 45 | 46 | func (it *prefixIteratorFlat) reset() { 47 | it.seek = false 48 | it.done = false 49 | } 50 | 51 | func (it *prefixIteratorFlat) Reset() { 52 | it.base.Reset() 53 | it.reset() 54 | } 55 | 56 | func (it *prefixIteratorFlat) WithPrefix(pref flat.Key) flat.Iterator { 57 | if len(pref) == 0 { 58 | return it.base 59 | } 60 | it.pref = pref 61 | it.reset() 62 | return it 63 | } 64 | 65 | func (it *prefixIteratorFlat) Next(ctx context.Context) bool { 66 | if it.done { 67 | return false 68 | } 69 | if !it.seek { 70 | found := flat.Seek(ctx, it.base, it.pref) 71 | it.seek = true 72 | if !found { 73 | it.done = true 74 | return false 75 | } 76 | } else { 77 | if !it.base.Next(ctx) { 78 | it.done = true 79 | return false 80 | } 81 | } 82 | key := it.base.Key() 83 | if bytes.HasPrefix(key, it.pref) { 84 | return true 85 | } 86 | // keys are sorted, and we reached the end of the prefix 87 | it.done = true 88 | return false 89 | } 90 | 91 | func (it *prefixIteratorFlat) Err() error { 92 | return it.base.Err() 93 | } 94 | 95 | func (it *prefixIteratorFlat) Close() error { 96 | return it.base.Close() 97 | } 98 | 99 | func (it *prefixIteratorFlat) Key() flat.Key { 100 | return it.base.Key() 101 | } 102 | 103 | func (it *prefixIteratorFlat) Val() flat.Value { 104 | return it.base.Val() 105 | } 106 | -------------------------------------------------------------------------------- /kv/options/prefix_kv.go: -------------------------------------------------------------------------------- 1 | package options 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/hidal-go/hidalgo/kv" 7 | "github.com/hidal-go/hidalgo/kv/flat" 8 | ) 9 | 10 | // WithPrefixKV returns IteratorOption that limits scanned key to a given binary prefix. 11 | // Store implementations can optimize this by implementing kv.PrefixIterator. 12 | func WithPrefixKV(pref kv.Key) IteratorOption { 13 | return PrefixKV{Pref: pref} 14 | } 15 | 16 | // PrefixKV implements IteratorOption. See WithPrefixKV. 17 | type PrefixKV struct { 18 | Pref kv.Key 19 | } 20 | 21 | func (opt PrefixKV) ApplyKV(it kv.Iterator) kv.Iterator { 22 | if it, ok := it.(kv.PrefixIterator); ok { 23 | return it.WithPrefix(opt.Pref) 24 | } 25 | return &prefixIteratorKV{base: it, pref: opt.Pref} 26 | } 27 | 28 | func (opt PrefixKV) ApplyFlat(it flat.Iterator) flat.Iterator { 29 | pref := flat.KeyEscape(opt.Pref) 30 | if it, ok := it.(flat.PrefixIterator); ok { 31 | return it.WithPrefix(pref) 32 | } 33 | return &prefixIteratorFlat{base: it, pref: pref} 34 | } 35 | 36 | var _ kv.PrefixIterator = &prefixIteratorKV{} 37 | 38 | type prefixIteratorKV struct { 39 | base kv.Iterator 40 | pref kv.Key 41 | seek bool 42 | done bool 43 | } 44 | 45 | func (it *prefixIteratorKV) reset() { 46 | it.seek = false 47 | it.done = false 48 | } 49 | 50 | func (it *prefixIteratorKV) Reset() { 51 | it.base.Reset() 52 | it.reset() 53 | } 54 | 55 | func (it *prefixIteratorKV) WithPrefix(pref kv.Key) kv.Iterator { 56 | if len(pref) == 0 { 57 | return it.base 58 | } 59 | it.pref = pref 60 | it.reset() 61 | return it 62 | } 63 | 64 | func (it *prefixIteratorKV) Next(ctx context.Context) bool { 65 | if it.done { 66 | return false 67 | } 68 | if !it.seek { 69 | found := kv.Seek(ctx, it.base, it.pref) 70 | it.seek = true 71 | if !found { 72 | it.done = true 73 | return false 74 | } 75 | } else { 76 | if !it.base.Next(ctx) { 77 | it.done = true 78 | return false 79 | } 80 | } 81 | key := it.base.Key() 82 | if key.HasPrefix(it.pref) { 83 | return true 84 | } 85 | // keys are sorted, and we reached the end of the prefix 86 | it.done = true 87 | return false 88 | } 89 | 90 | func (it *prefixIteratorKV) Err() error { 91 | return it.base.Err() 92 | } 93 | 94 | func (it *prefixIteratorKV) Close() error { 95 | return it.base.Close() 96 | } 97 | 98 | func (it *prefixIteratorKV) Key() kv.Key { 99 | return it.base.Key() 100 | } 101 | 102 | func (it *prefixIteratorKV) Val() kv.Value { 103 | return it.base.Val() 104 | } 105 | -------------------------------------------------------------------------------- /kv/registry.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "sort" 5 | 6 | "github.com/hidal-go/hidalgo/base" 7 | ) 8 | 9 | // OpenPathFunc is a function for opening a database given a path. 10 | type OpenPathFunc func(path string) (KV, error) 11 | 12 | // Registration is an information about the database driver. 13 | type Registration struct { 14 | base.Registration 15 | OpenPath OpenPathFunc 16 | } 17 | 18 | var registry = make(map[string]Registration) 19 | 20 | // Register globally registers a database driver. 21 | func Register(reg Registration) { 22 | if reg.Name == "" { 23 | panic("name cannot be empty") 24 | } else if _, ok := registry[reg.Name]; ok { 25 | panic(base.ErrRegistered{Name: reg.Name}) 26 | } 27 | registry[reg.Name] = reg 28 | } 29 | 30 | // List enumerates all globally registered database drivers. 31 | func List() []Registration { 32 | out := make([]Registration, 0, len(registry)) 33 | for _, r := range registry { 34 | out = append(out, r) 35 | } 36 | sort.Slice(out, func(i, j int) bool { 37 | return out[i].Name < out[j].Name 38 | }) 39 | return out 40 | } 41 | 42 | // ByName returns a registered database driver by it's name. 43 | func ByName(name string) *Registration { 44 | r, ok := registry[name] 45 | if !ok { 46 | return nil 47 | } 48 | return &r 49 | } 50 | -------------------------------------------------------------------------------- /legacy/nosql/all/all.go: -------------------------------------------------------------------------------- 1 | package all 2 | 3 | import ( 4 | _ "github.com/hidal-go/hidalgo/legacy/nosql/couch" 5 | _ "github.com/hidal-go/hidalgo/legacy/nosql/elastic" 6 | _ "github.com/hidal-go/hidalgo/legacy/nosql/mongo" 7 | ) 8 | -------------------------------------------------------------------------------- /legacy/nosql/couch/couch.go: -------------------------------------------------------------------------------- 1 | //go:build !js 2 | // +build !js 3 | 4 | package couch 5 | 6 | import ( 7 | "context" 8 | 9 | _ "github.com/go-kivik/couchdb" // The CouchDB driver 10 | 11 | "github.com/hidal-go/hidalgo/base" 12 | "github.com/hidal-go/hidalgo/legacy/nosql" 13 | ) 14 | 15 | const ( 16 | NameCouch = "couch" 17 | DriverCouch = "couch" 18 | ) 19 | 20 | func init() { 21 | nosql.Register(nosql.Registration{ 22 | Registration: base.Registration{ 23 | Name: NameCouch, Title: "CouchDB", 24 | Local: false, Volatile: false, 25 | }, 26 | Traits: Traits(), 27 | New: CreateCouch, Open: OpenCouch, 28 | }) 29 | } 30 | 31 | func CreateCouch(ctx context.Context, addr, ns string, opt nosql.Options) (nosql.Database, error) { 32 | return Dial(ctx, true, DriverCouch, addr, ns, opt) 33 | } 34 | 35 | func OpenCouch(ctx context.Context, addr, ns string, opt nosql.Options) (nosql.Database, error) { 36 | return Dial(ctx, false, DriverCouch, addr, ns, opt) 37 | } 38 | -------------------------------------------------------------------------------- /legacy/nosql/couch/couch_test.go: -------------------------------------------------------------------------------- 1 | //go:build !js 2 | // +build !js 3 | 4 | package couch_test 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/hidal-go/hidalgo/legacy/nosql/couch" 10 | _ "github.com/hidal-go/hidalgo/legacy/nosql/couch/test" 11 | "github.com/hidal-go/hidalgo/legacy/nosql/nosqltest" 12 | ) 13 | 14 | func TestCouch(t *testing.T) { 15 | nosqltest.Test(t, couch.NameCouch) 16 | } 17 | 18 | func BenchmarkCouch(b *testing.B) { 19 | nosqltest.Benchmark(b, couch.NameCouch) 20 | } 21 | -------------------------------------------------------------------------------- /legacy/nosql/couch/ouch_test.go: -------------------------------------------------------------------------------- 1 | package couch 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestSelector(t *testing.T) { 10 | q := ouchQuery{"selector": make(map[string]interface{})} 11 | in := []interface{}{"a", "b"} 12 | q.putSelector(idField, map[string]interface{}{"$in": in}) 13 | q.putSelector(idField, map[string]interface{}{"$gt": "a"}) 14 | require.Equal(t, ouchQuery{ 15 | "selector": map[string]interface{}{ 16 | idField: map[string]interface{}{ 17 | "$in": in, 18 | "$gt": "a", 19 | }, 20 | }, 21 | }, q) 22 | } 23 | -------------------------------------------------------------------------------- /legacy/nosql/couch/pouch.go: -------------------------------------------------------------------------------- 1 | // +build js 2 | 3 | package couch 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/hidal-go/hidalgo/base" 9 | "github.com/hidal-go/hidalgo/legacy/nosql" 10 | 11 | _ "github.com/go-kivik/pouchdb" // The PouchDB driver 12 | ) 13 | 14 | const ( 15 | NamePouch = "pouch" 16 | DriverPouch = "couch" 17 | ) 18 | 19 | func init() { 20 | nosql.Register(nosql.Registration{ 21 | Registration: base.Registration{ 22 | Name: NamePouch, Title: DriverPouch, 23 | Local: true, Volatile: false, 24 | }, 25 | Traits: Traits(), 26 | New: CreatePouch, Open: OpenPouch, 27 | }) 28 | } 29 | 30 | func CreatePouch(ctx context.Context, addr string, ns string, opt nosql.Options) (nosql.Database, error) { 31 | return Dial(ctx, true, DriverPouch, addr, ns, opt) 32 | } 33 | 34 | func OpenPouch(ctx context.Context, addr string, ns string, opt nosql.Options) (nosql.Database, error) { 35 | return Dial(ctx, false, DriverPouch, addr, ns, opt) 36 | } 37 | -------------------------------------------------------------------------------- /legacy/nosql/couch/pouch_test.go: -------------------------------------------------------------------------------- 1 | // +build js 2 | 3 | // To run "gopherjs test -v" you must "npm install pouchdb" and "npm install pouchdb-find" for queries. 4 | 5 | package couch_test 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/hidal-go/hidalgo/legacy/nosql/couch" 11 | _ "github.com/hidal-go/hidalgo/legacy/nosql/couch/test" 12 | "github.com/hidal-go/hidalgo/legacy/nosql/nosqltest" 13 | ) 14 | 15 | func TestPouch(t *testing.T) { 16 | nosqltest.Test(t, couch.NamePouch) 17 | } 18 | 19 | func BenchmarkPouch(b *testing.B) { 20 | nosqltest.Benchmark(b, couch.NamePouch) 21 | } 22 | -------------------------------------------------------------------------------- /legacy/nosql/couch/test/couchtest.go: -------------------------------------------------------------------------------- 1 | //go:build !js 2 | // +build !js 3 | 4 | package couchtest 5 | 6 | import ( 7 | "context" 8 | "testing" 9 | 10 | "github.com/ory/dockertest" 11 | 12 | "github.com/hidal-go/hidalgo/legacy/nosql" 13 | "github.com/hidal-go/hidalgo/legacy/nosql/couch" 14 | "github.com/hidal-go/hidalgo/legacy/nosql/nosqltest" 15 | ) 16 | 17 | func init() { 18 | const vers = "2" 19 | nosqltest.Register(couch.NameCouch, nosqltest.Version{ 20 | Name: vers, Factory: CouchVersion(vers), 21 | }) 22 | } 23 | 24 | func CouchVersion(vers string) nosqltest.Database { 25 | return nosqltest.Database{ 26 | Traits: couch.Traits(), 27 | Run: func(t testing.TB) nosql.Database { 28 | pool, err := dockertest.NewPool("") 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | 33 | cont, err := pool.Run("couchdb", vers, []string{ 34 | "COUCHDB_USER=test", 35 | "COUCHDB_PASSWORD=test", 36 | }) 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | t.Cleanup(func() { 41 | _ = cont.Close() 42 | }) 43 | 44 | ctx := context.Background() 45 | 46 | addr := cont.GetHostPort("5984/tcp") 47 | addr = "http://test:test@" + addr + "/test" 48 | err = pool.Retry(func() error { 49 | cli, _, err := couch.DialDriver(ctx, couch.DriverCouch, addr, "test") 50 | if err != nil { 51 | return err 52 | } 53 | _, err = cli.Version(ctx) 54 | return err 55 | }) 56 | if err != nil { 57 | t.Fatal(err) 58 | } 59 | 60 | qs, err := couch.Dial(ctx, true, couch.DriverCouch, addr, "test", nil) 61 | if err != nil { 62 | t.Fatal(err) 63 | } 64 | t.Cleanup(func() { 65 | _ = qs.Close() 66 | }) 67 | return qs 68 | }, 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /legacy/nosql/couch/test/pouchtest.go: -------------------------------------------------------------------------------- 1 | //go:build js 2 | // +build js 3 | 4 | package couchtest 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "io/ioutil" 10 | "math/rand" 11 | "os" 12 | "testing" 13 | 14 | "github.com/go-kivik/kivik" 15 | "github.com/hidal-go/hidalgo/legacy/nosql" 16 | "github.com/hidal-go/hidalgo/legacy/nosql/couch" 17 | "github.com/hidal-go/hidalgo/legacy/nosql/nosqltest" 18 | ) 19 | 20 | func init() { 21 | nosqltest.Register(couch.NamePouch, nosqltest.Version{ 22 | Name: "pouch", Factory: Pouch(), 23 | }) 24 | } 25 | 26 | func Pouch() nosqltest.Database { 27 | return nosqltest.Database{ 28 | Traits: couch.Traits(), 29 | Run: func(t testing.TB) nosql.Database { 30 | dir, err := ioutil.TempDir("", "pouch-") 31 | if err != nil { 32 | t.Fatal("failed to make temp dir:", err) 33 | } 34 | t.Cleanup(func() { 35 | if err := os.RemoveAll(dir); err != nil { // remove the test data 36 | t.Fatal(err) 37 | } 38 | }) 39 | 40 | ctx := context.Background() 41 | name := fmt.Sprintf("db-%d", rand.Int()) 42 | 43 | qs, err := couch.Dial(ctx, false, couch.DriverPouch, dir+"/"+name, name, nil) 44 | if err != nil { 45 | os.RemoveAll(dir) 46 | t.Fatal(err) 47 | } 48 | t.Cleanup(func() { 49 | qs.Close() 50 | if c, err := kivik.New(couch.DriverPouch, dir); err == nil { 51 | _ = c.DestroyDB(ctx, name) 52 | } 53 | }) 54 | 55 | return qs 56 | }, 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /legacy/nosql/couch/values.go: -------------------------------------------------------------------------------- 1 | package couch 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "strings" 7 | "time" 8 | 9 | "github.com/hidal-go/hidalgo/legacy/nosql" 10 | ) 11 | 12 | const ( 13 | keySeparator = "|" 14 | ) 15 | 16 | // toOuchValue serializes nosql.Value -> native json values. 17 | func toOuchValue(v nosql.Value) interface{} { 18 | switch v := v.(type) { 19 | case nil: 20 | return nil 21 | case nosql.Strings: 22 | return []string(v) 23 | case nosql.String: 24 | return string(v) 25 | case nosql.Int: 26 | return int64(v) 27 | case nosql.Float: 28 | return float64(v) 29 | case nosql.Bool: 30 | return bool(v) 31 | case nosql.Time: 32 | return time.Time(v).UTC().Format(time.RFC3339Nano) 33 | case nosql.Bytes: 34 | return base64.StdEncoding.EncodeToString(v) 35 | default: 36 | panic(fmt.Errorf("unsupported type: %T", v)) 37 | } 38 | } 39 | 40 | func toOuchDoc(col, id, rev string, d nosql.Document) map[string]interface{} { 41 | if d == nil { 42 | return nil 43 | } 44 | m := make(map[string]interface{}) 45 | if col != "" { 46 | m[collectionField] = col 47 | } 48 | if id != "" { 49 | m[idField] = id 50 | } 51 | if rev != "" { 52 | m[revField] = rev 53 | } 54 | 55 | for k, v := range d { 56 | if len(k) == 0 { 57 | continue 58 | } 59 | if sub, ok := v.(nosql.Document); ok { 60 | for sk, sv := range sub { 61 | path := k + keySeparator + sk 62 | m[path] = toOuchValue(sv) 63 | } 64 | } else { 65 | m[k] = toOuchValue(v) 66 | } 67 | } 68 | 69 | return m 70 | } 71 | 72 | func fromOuchValue(v interface{}) nosql.Value { 73 | switch v := v.(type) { 74 | case nil: 75 | return nil 76 | case []interface{}: 77 | out := make([]string, 0, len(v)) 78 | for _, o := range v { 79 | s, ok := o.(string) 80 | if !ok { 81 | panic(fmt.Errorf("unexpected type in array: %T", o)) 82 | } 83 | out = append(out, s) 84 | } 85 | return nosql.Strings(out) 86 | case string: 87 | return nosql.String(v) 88 | case float64: 89 | return nosql.Float(v) 90 | case bool: 91 | return nosql.Bool(v) 92 | default: 93 | panic(fmt.Errorf("unsupported type: %T", v)) 94 | } 95 | } 96 | 97 | func fromOuchDoc(d map[string]interface{}) nosql.Document { 98 | if d == nil { 99 | return nil 100 | } 101 | m := make(nosql.Document, len(d)) 102 | for k, v := range d { 103 | switch k { 104 | case "", idField, revField, collectionField: 105 | continue // don't pass these fields back to nosql 106 | } 107 | if k[0] != ' ' { // ignore any other ouch driver internal keys 108 | if path := strings.Split(k, keySeparator); len(path) > 1 { 109 | if len(path) != 2 { 110 | panic("nosql.Document nesting too deep") 111 | } 112 | // we have a sub-document 113 | if _, found := m[path[0]]; !found { 114 | m[path[0]] = make(nosql.Document) 115 | } 116 | m[path[0]].(nosql.Document)[path[1]] = fromOuchValue(v) 117 | } else { 118 | m[k] = fromOuchValue(v) 119 | } 120 | } 121 | } 122 | 123 | if len(m) == 0 { 124 | return nil 125 | } 126 | 127 | return m 128 | } 129 | -------------------------------------------------------------------------------- /legacy/nosql/elastic/elastic_test.go: -------------------------------------------------------------------------------- 1 | package elastic_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/hidal-go/hidalgo/legacy/nosql/elastic" 7 | _ "github.com/hidal-go/hidalgo/legacy/nosql/elastic/test" 8 | "github.com/hidal-go/hidalgo/legacy/nosql/nosqltest" 9 | ) 10 | 11 | func TestElastic(t *testing.T) { 12 | if testing.Short() { 13 | t.SkipNow() 14 | } 15 | nosqltest.Test(t, elastic.Name) 16 | } 17 | 18 | func BenchmarkElastic(b *testing.B) { 19 | nosqltest.Benchmark(b, elastic.Name) 20 | } 21 | -------------------------------------------------------------------------------- /legacy/nosql/elastic/test/elastictest.go: -------------------------------------------------------------------------------- 1 | package elastictest 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/ory/dockertest" 8 | edriver "gopkg.in/olivere/elastic.v5" 9 | 10 | "github.com/hidal-go/hidalgo/legacy/nosql" 11 | "github.com/hidal-go/hidalgo/legacy/nosql/elastic" 12 | "github.com/hidal-go/hidalgo/legacy/nosql/nosqltest" 13 | ) 14 | 15 | var versions = []string{ 16 | "6.2.4", 17 | } 18 | 19 | func init() { 20 | var vers []nosqltest.Version 21 | for _, v := range versions { 22 | vers = append(vers, nosqltest.Version{ 23 | Name: v, Factory: ElasticVersion(v), 24 | }) 25 | } 26 | nosqltest.Register(elastic.Name, vers...) 27 | } 28 | 29 | func ElasticVersion(vers string) nosqltest.Database { 30 | return nosqltest.Database{ 31 | Traits: elastic.Traits(), 32 | Run: func(t testing.TB) nosql.Database { 33 | name := "docker.elastic.co/elasticsearch/elasticsearch" 34 | 35 | pool, err := dockertest.NewPool("") 36 | if err != nil { 37 | t.Fatal(err) 38 | } 39 | 40 | cont, err := pool.Run(name, vers, nil) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | t.Cleanup(func() { 45 | _ = cont.Close() 46 | }) 47 | 48 | // Running this command might be necessary on the host: 49 | // sysctl -w vm.max_map_count=262144 50 | 51 | const port = "9200/tcp" 52 | addr := "http://" + cont.GetHostPort(port) 53 | ctx := context.Background() 54 | 55 | err = pool.Retry(func() error { 56 | cli, err := edriver.NewClient(edriver.SetURL(addr)) 57 | if err != nil { 58 | return err 59 | } 60 | _, _, err = cli.Ping(addr).Do(ctx) 61 | return err 62 | }) 63 | if err != nil { 64 | t.Fatal(err) 65 | } 66 | 67 | db, err := elastic.Dial(ctx, addr, "test", nil) 68 | if err != nil { 69 | t.Fatal(addr, err) 70 | } 71 | t.Cleanup(func() { 72 | _ = db.Close() 73 | }) 74 | return db 75 | }, 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /legacy/nosql/mongo/mongo_test.go: -------------------------------------------------------------------------------- 1 | package mongo_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/hidal-go/hidalgo/legacy/nosql/mongo" 7 | _ "github.com/hidal-go/hidalgo/legacy/nosql/mongo/test" 8 | "github.com/hidal-go/hidalgo/legacy/nosql/nosqltest" 9 | ) 10 | 11 | func TestMongo(t *testing.T) { 12 | nosqltest.Test(t, mongo.Name) 13 | } 14 | 15 | func BenchmarkMongo(b *testing.B) { 16 | nosqltest.Benchmark(b, mongo.Name) 17 | } 18 | -------------------------------------------------------------------------------- /legacy/nosql/mongo/test/mongotest.go: -------------------------------------------------------------------------------- 1 | package mongotest 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/ory/dockertest" 9 | gomongo "go.mongodb.org/mongo-driver/mongo" 10 | "go.mongodb.org/mongo-driver/mongo/options" 11 | 12 | "github.com/hidal-go/hidalgo/legacy/nosql" 13 | "github.com/hidal-go/hidalgo/legacy/nosql/mongo" 14 | "github.com/hidal-go/hidalgo/legacy/nosql/nosqltest" 15 | ) 16 | 17 | const vers = "3" 18 | 19 | func init() { 20 | nosqltest.Register(mongo.Name, nosqltest.Version{ 21 | Name: vers, Factory: MongoVersion(vers), 22 | }) 23 | } 24 | 25 | func MongoVersion(vers string) nosqltest.Database { 26 | return nosqltest.Database{ 27 | Traits: mongo.Traits(), 28 | Run: func(t testing.TB) nosql.Database { 29 | pool, err := dockertest.NewPool("") 30 | if err != nil { 31 | t.Fatal(err) 32 | } 33 | 34 | cont, err := pool.Run("mongo", vers, nil) 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | t.Cleanup(func() { 39 | _ = cont.Close() 40 | }) 41 | 42 | addr := fmt.Sprintf("mongodb://%s", cont.GetHostPort("27017/tcp")) 43 | ctx := context.Background() 44 | err = pool.Retry(func() error { 45 | sess, err := gomongo.NewClient(options.Client().ApplyURI(addr)) 46 | if err != nil { 47 | return err 48 | } 49 | defer sess.Disconnect(ctx) 50 | 51 | err = sess.Connect(ctx) 52 | 53 | if err != nil { 54 | return err 55 | } 56 | return nil 57 | }) 58 | if err != nil { 59 | t.Fatal(err) 60 | } 61 | qs, err := mongo.Dial(ctx, addr, "test", nil) 62 | if err != nil { 63 | t.Fatal(err) 64 | } 65 | t.Cleanup(func() { 66 | _ = qs.Close() 67 | }) 68 | return qs 69 | }, 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /legacy/nosql/nosql.go: -------------------------------------------------------------------------------- 1 | package nosql 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "regexp" 8 | 9 | "github.com/pborman/uuid" 10 | ) 11 | 12 | var ErrNotFound = errors.New("not found") 13 | 14 | type Options map[string]interface{} 15 | 16 | func (o Options) GetString(key, def string) string { 17 | v, ok := o[key].(string) 18 | if !ok { 19 | return def 20 | } 21 | return v 22 | } 23 | 24 | type Traits struct { 25 | Number32 bool // store is limited to 32 bit precision 26 | FloatToInt bool // database silently converts all float values to ints (if possible) 27 | IntToFloat bool // database always converts all int values to floats 28 | TimeInMs bool 29 | PageSize int // result page size for pagination 30 | } 31 | 32 | // Key is a set of values that describe primary key of document. 33 | type Key []string 34 | 35 | // Value converts a Key to a value that can be stored in the database. 36 | func (k Key) Value() Value { 37 | return Strings(k) 38 | } 39 | 40 | // GenKey generates a unique key (with one field). 41 | func GenKey() Key { 42 | return Key{uuid.NewUUID().String()} 43 | } 44 | 45 | // KeyFrom extracts a set of fields as a Key from Document. 46 | func KeyFrom(fields []string, doc Document) Key { 47 | key := make(Key, 0, len(fields)) 48 | for _, f := range fields { 49 | if s, ok := doc[f].(String); ok { 50 | key = append(key, string(s)) 51 | } 52 | } 53 | return key 54 | } 55 | 56 | // Database is a minimal interface for NoSQL database implementations. 57 | type Database interface { 58 | // Insert creates a document with a given key in a given collection. 59 | // Key can be nil meaning that implementation should generate a unique key for the item. 60 | // It returns the key that was generated, or the same key that was passed to it. 61 | Insert(ctx context.Context, col string, key Key, d Document) (Key, error) 62 | // FindByKey finds a document by it's Key. It returns ErrNotFound if document not exists. 63 | FindByKey(ctx context.Context, col string, key Key) (Document, error) 64 | // Query starts construction of a new query for a specified collection. 65 | Query(col string) Query 66 | // Update starts construction of document update request for a specified document and collection. 67 | Update(col string, key Key) Update 68 | // Delete starts construction of document delete request. 69 | Delete(col string) Delete 70 | // EnsureIndex creates or updates indexes on the collection to match it's arguments. 71 | // It should create collection if it not exists. Primary index is guaranteed to be of StringExact type. 72 | EnsureIndex(ctx context.Context, col string, primary Index, secondary []Index) error 73 | // Close closes the database connection. 74 | Close() error 75 | } 76 | 77 | // FilterOp is a comparison operation type used for value filters. 78 | type FilterOp int 79 | 80 | func (op FilterOp) String() string { 81 | name := "" 82 | switch op { 83 | case Equal: 84 | name = "Equal" 85 | case NotEqual: 86 | name = "NotEqual" 87 | case GT: 88 | name = "GT" 89 | case GTE: 90 | name = "GTE" 91 | case LT: 92 | name = "LT" 93 | case LTE: 94 | name = "LTE" 95 | default: 96 | return fmt.Sprintf("FilterOp(%d)", int(op)) 97 | } 98 | return name 99 | } 100 | 101 | func (op FilterOp) GoString() string { 102 | return "nosql." + op.String() 103 | } 104 | 105 | const ( 106 | Equal = FilterOp(iota + 1) 107 | NotEqual 108 | GT 109 | GTE 110 | LT 111 | LTE 112 | Regexp 113 | ) 114 | 115 | // FieldFilter represents a single field comparison operation. 116 | type FieldFilter struct { 117 | Path []string // path is a path to specific field in the document 118 | Filter FilterOp // comparison operation 119 | Value Value // value that will be compared with field of the document 120 | } 121 | 122 | func (f FieldFilter) Matches(doc Document) bool { 123 | path := f.Path 124 | var val Value = doc 125 | 126 | if f.Filter == NotEqual { 127 | // NotEqual is special - it allows parent fields to not exist 128 | for len(path) > 0 { 129 | d, ok := val.(Document) 130 | if !ok { 131 | return true 132 | } 133 | v, ok := d[path[0]] 134 | if !ok { 135 | return true 136 | } 137 | val, path = v, path[1:] 138 | } 139 | return !ValuesEqual(val, f.Value) 140 | } 141 | 142 | for len(path) > 0 { 143 | d, ok := val.(Document) 144 | if !ok { 145 | return false 146 | } 147 | v, ok := d[path[0]] 148 | if !ok { 149 | return false 150 | } 151 | val, path = v, path[1:] 152 | } 153 | switch f.Filter { 154 | case Equal: 155 | return ValuesEqual(val, f.Value) 156 | case GT, GTE, LT, LTE: 157 | dn := CompareValues(val, f.Value) 158 | switch f.Filter { 159 | case GT: 160 | return dn > 0 161 | case GTE: 162 | return dn >= 0 163 | case LT: 164 | return dn < 0 165 | case LTE: 166 | return dn <= 0 167 | } 168 | case Regexp: 169 | pattern, ok := f.Value.(String) 170 | if !ok { 171 | return false 172 | } 173 | s, ok := val.(String) 174 | if !ok { 175 | return false 176 | } 177 | ok, _ = regexp.MatchString(string(pattern), string(s)) 178 | return ok 179 | } 180 | panic(fmt.Errorf("unsupported operation: %v", f.Filter)) 181 | } 182 | 183 | // Query is a query builder object. 184 | type Query interface { 185 | // WithFields adds specified filters to the query. 186 | WithFields(filters ...FieldFilter) Query 187 | // Limit limits a maximal number of results returned. 188 | Limit(n int) Query 189 | 190 | // Count executes query and returns a number of items that matches it. 191 | Count(ctx context.Context) (int64, error) 192 | // One executes query and returns first document from it. 193 | One(ctx context.Context) (Document, error) 194 | // Iterate starts an iteration over query results. 195 | Iterate(ctx context.Context) DocIterator 196 | } 197 | 198 | // Update is an update request builder. 199 | type Update interface { 200 | // Inc increments document field with a given amount. Will also increment upserted document. 201 | Inc(field string, dn int) Update 202 | // Upsert sets a document that will be inserted in case original object does not exists already. 203 | // It should omit fields used by Inc - they will be added automatically. 204 | Upsert(d Document) Update 205 | // Do executes update request. 206 | Do(ctx context.Context) error 207 | } 208 | 209 | // Update is a batch delete request builder. 210 | type Delete interface { 211 | // WithFields adds specified filters to select document for deletion. 212 | WithFields(filters ...FieldFilter) Delete 213 | // Keys limits a set of documents to delete to ones with keys specified. 214 | // Delete still uses provided filters, thus it will not delete objects with these keys if they do not pass filters. 215 | Keys(keys ...Key) Delete 216 | // Do executes batch delete. 217 | Do(ctx context.Context) error 218 | } 219 | 220 | // DocIterator is an iterator over a list of documents. 221 | type DocIterator interface { 222 | // Next advances an iterator to the next document. 223 | Next(ctx context.Context) bool 224 | // Err returns a last encountered error. 225 | Err() error 226 | // Close frees all resources associated with iterator. 227 | Close() error 228 | // Key returns a key of current document. 229 | Key() Key 230 | // Doc returns current document. 231 | Doc() Document 232 | } 233 | 234 | // BatchInsert returns a streaming writer for database or emulates it if database has no support for batch inserts. 235 | func BatchInsert(db Database, col string) DocWriter { 236 | if bi, ok := db.(BatchInserter); ok { 237 | return bi.BatchInsert(col) 238 | } 239 | return &seqInsert{db: db, col: col} 240 | } 241 | 242 | type seqInsert struct { 243 | db Database 244 | col string 245 | keys []Key 246 | err error 247 | } 248 | 249 | func (w *seqInsert) WriteDoc(ctx context.Context, key Key, d Document) error { 250 | key, err := w.db.Insert(ctx, w.col, key, d) 251 | if err != nil { 252 | w.err = err 253 | return err 254 | } 255 | w.keys = append(w.keys, key) 256 | return nil 257 | } 258 | 259 | func (w *seqInsert) Flush(ctx context.Context) error { 260 | return w.err 261 | } 262 | 263 | func (w *seqInsert) Keys() []Key { 264 | return w.keys 265 | } 266 | 267 | func (w *seqInsert) Close() error { 268 | return w.err 269 | } 270 | 271 | // DocWriter is an interface for writing documents in streaming manner. 272 | type DocWriter interface { 273 | // WriteDoc prepares document to be written. Write becomes valid only after Flush. 274 | WriteDoc(ctx context.Context, key Key, d Document) error 275 | // Flush waits for all writes to complete. 276 | Flush(ctx context.Context) error 277 | // Keys returns a list of already inserted documents. 278 | // Might be less then a number of written documents until Flush is called. 279 | Keys() []Key 280 | // Close closes writer and discards any unflushed documents. 281 | Close() error 282 | } 283 | 284 | // BatchInserter is an optional interface for databases that can insert documents in batches. 285 | type BatchInserter interface { 286 | BatchInsert(col string) DocWriter 287 | } 288 | 289 | // IndexType is a type of index for collection. 290 | type IndexType int 291 | 292 | const ( 293 | IndexAny = IndexType(iota) 294 | StringExact // exact match for string values (usually a hash index) 295 | 296 | // TODO: StringFulltext 297 | // TODO: IntIndex 298 | // TODO: FloatIndex 299 | // TODO: TimeIndex 300 | ) 301 | 302 | // Index is an index for a collection of documents. 303 | type Index struct { 304 | Fields []string // an ordered set of fields used in index 305 | Type IndexType 306 | } 307 | -------------------------------------------------------------------------------- /legacy/nosql/nosqltest/all/alltest.go: -------------------------------------------------------------------------------- 1 | package alltest 2 | 3 | import ( 4 | _ "github.com/hidal-go/hidalgo/legacy/nosql/couch/test" 5 | _ "github.com/hidal-go/hidalgo/legacy/nosql/elastic/test" 6 | _ "github.com/hidal-go/hidalgo/legacy/nosql/mongo/test" 7 | ) 8 | -------------------------------------------------------------------------------- /legacy/nosql/nosqltest/registry.go: -------------------------------------------------------------------------------- 1 | package nosqltest 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | 7 | "github.com/hidal-go/hidalgo/legacy/nosql" 8 | ) 9 | 10 | type Registration struct { 11 | nosql.Registration 12 | Versions []Version 13 | } 14 | 15 | type Version struct { 16 | Name string 17 | Factory Database 18 | } 19 | 20 | var registry = make(map[string][]Version) 21 | 22 | // Register globally registers a database driver. 23 | func Register(name string, vers ...Version) { 24 | if name == "" { 25 | panic("name cannot be empty") 26 | } else if len(vers) == 0 { 27 | panic("at least one version should be specified") 28 | } else if r := nosql.ByName(name); r == nil { 29 | panic("name is not registered") 30 | } 31 | vers = append([]Version{}, vers...) 32 | sort.Slice(vers, func(i, j int) bool { 33 | return vers[i].Name < vers[j].Name 34 | }) 35 | registry[name] = vers 36 | } 37 | 38 | // List enumerates all globally registered database drivers. 39 | func List() []Registration { 40 | out := make([]Registration, 0, len(registry)) 41 | for name, vers := range registry { 42 | out = append(out, Registration{ 43 | Registration: *nosql.ByName(name), 44 | Versions: append([]Version{}, vers...), 45 | }) 46 | } 47 | sort.Slice(out, func(i, j int) bool { 48 | return out[i].Name < out[j].Name 49 | }) 50 | return out 51 | } 52 | 53 | // ByName returns a registered database driver by it's name. 54 | func ByName(name string) *Registration { 55 | vers, ok := registry[name] 56 | if !ok { 57 | return nil 58 | } 59 | return &Registration{ 60 | Registration: *nosql.ByName(name), 61 | Versions: append([]Version{}, vers...), 62 | } 63 | } 64 | 65 | func allNames() []string { 66 | var names []string 67 | for name := range registry { 68 | names = append(names, name) 69 | } 70 | return names 71 | } 72 | 73 | func runT(t *testing.T, test func(t *testing.T, run Database), name string) { 74 | for _, v := range ByName(name).Versions { 75 | t.Run(v.Name, func(tt *testing.T) { test(tt, v.Factory) }) 76 | } 77 | } 78 | 79 | func runB(b *testing.B, bench func(b *testing.B, run Database), name string) { 80 | for _, v := range ByName(name).Versions { 81 | b.Run(v.Name, func(bb *testing.B) { bench(bb, v.Factory) }) 82 | } 83 | } 84 | 85 | func RunTest(t *testing.T, test func(t *testing.T, run Database), names ...string) { 86 | for _, name := range names { 87 | if _, ok := registry[name]; !ok { 88 | panic("not registered: " + name) 89 | } 90 | } 91 | if len(names) == 0 { 92 | names = allNames() 93 | } 94 | for _, name := range names { 95 | t.Run(name, func(tt *testing.T) { runT(tt, test, name) }) 96 | } 97 | } 98 | 99 | func RunBenchmark(b *testing.B, bench func(b *testing.B, run Database), names ...string) { 100 | for _, name := range names { 101 | if _, ok := registry[name]; !ok { 102 | panic("not registered: " + name) 103 | } 104 | } 105 | if len(names) == 0 { 106 | names = allNames() 107 | } 108 | for _, name := range names { 109 | b.Run(name, func(bb *testing.B) { runB(bb, bench, name) }) 110 | } 111 | } 112 | 113 | func Test(t *testing.T, names ...string) { 114 | RunTest(t, TestNoSQL, names...) 115 | } 116 | 117 | func Benchmark(b *testing.B, names ...string) { 118 | RunBenchmark(b, BenchmarkNoSQL, names...) 119 | } 120 | -------------------------------------------------------------------------------- /legacy/nosql/registry.go: -------------------------------------------------------------------------------- 1 | package nosql 2 | 3 | import ( 4 | "context" 5 | "sort" 6 | 7 | "github.com/hidal-go/hidalgo/base" 8 | ) 9 | 10 | // OpenFunc is a function for opening a database given a address and the database name. 11 | type OpenFunc func(ctx context.Context, addr, ns string, opt Options) (Database, error) 12 | 13 | // Registration is an information about the database driver. 14 | type Registration struct { 15 | base.Registration 16 | New, Open OpenFunc 17 | Traits Traits 18 | } 19 | 20 | var registry = make(map[string]Registration) 21 | 22 | // Register globally registers a database driver. 23 | func Register(reg Registration) { 24 | if reg.Name == "" { 25 | panic("name cannot be empty") 26 | } else if _, ok := registry[reg.Name]; ok { 27 | panic(base.ErrRegistered{Name: reg.Name}) 28 | } 29 | if reg.New == nil { 30 | reg.New = reg.Open 31 | } 32 | if reg.Open == nil { 33 | reg.Open = reg.New 34 | } 35 | registry[reg.Name] = reg 36 | } 37 | 38 | // List enumerates all globally registered database drivers. 39 | func List() []Registration { 40 | out := make([]Registration, 0, len(registry)) 41 | for _, r := range registry { 42 | out = append(out, r) 43 | } 44 | sort.Slice(out, func(i, j int) bool { 45 | return out[i].Name < out[j].Name 46 | }) 47 | return out 48 | } 49 | 50 | // ByName returns a registered database driver by it's name. 51 | func ByName(name string) *Registration { 52 | r, ok := registry[name] 53 | if !ok { 54 | return nil 55 | } 56 | return &r 57 | } 58 | -------------------------------------------------------------------------------- /legacy/nosql/value.go: -------------------------------------------------------------------------------- 1 | package nosql 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | "time" 7 | ) 8 | 9 | // Value is a interface that limits a set of types that nosql database can handle. 10 | type Value interface { 11 | isValue() 12 | } 13 | 14 | // Document is a type of item stored in nosql database. 15 | type Document map[string]Value 16 | 17 | func (Document) isValue() {} 18 | 19 | // String is an UTF8 string value. 20 | type String string 21 | 22 | func (String) isValue() {} 23 | 24 | // Int is an int value. 25 | // 26 | // Some databases might not distinguish Int value from Float. 27 | // In this case implementation will take care of converting it to a correct type. 28 | type Int int64 29 | 30 | func (Int) isValue() {} 31 | 32 | // Float is an floating point value. 33 | // 34 | // Some databases might not distinguish Int value from Float. 35 | // In this case the package will take care of converting it to a correct type. 36 | type Float float64 37 | 38 | func (Float) isValue() {} 39 | 40 | // Bool is a boolean value. 41 | type Bool bool 42 | 43 | func (Bool) isValue() {} 44 | 45 | // Time is a timestamp value. 46 | // 47 | // Some databases has no type to represent time values. 48 | // In this case string/json representation can be used and package will take care of converting it. 49 | type Time time.Time 50 | 51 | func (Time) isValue() {} 52 | 53 | // Bytes is a raw binary data. 54 | // 55 | // Some databases has no type to represent binary data. 56 | // In this case base64 representation can be used and package will take care of converting it. 57 | type Bytes []byte 58 | 59 | func (Bytes) isValue() {} 60 | 61 | // Strings is an array of strings. Used mostly to store Keys. 62 | type Strings []string 63 | 64 | func (Strings) isValue() {} 65 | 66 | // ValuesEqual returns true if values are strictly equal. 67 | func ValuesEqual(v1, v2 Value) bool { 68 | switch v1 := v2.(type) { 69 | case Document: 70 | v2, ok := v2.(Document) 71 | if !ok || len(v1) != len(v2) { 72 | return false 73 | } 74 | for k, s1 := range v1 { 75 | if s2, ok := v2[k]; !ok || !ValuesEqual(s1, s2) { 76 | return false 77 | } 78 | } 79 | return true 80 | case Strings: 81 | v2, ok := v2.(Strings) 82 | if !ok || len(v1) != len(v2) { 83 | return false 84 | } 85 | for i := range v1 { 86 | if v1[i] != v2[i] { 87 | return false 88 | } 89 | } 90 | return true 91 | case Bytes: 92 | v2, ok := v2.(Bytes) 93 | if !ok || len(v1) != len(v2) { 94 | return false 95 | } 96 | return bytes.Equal(v1, v2) 97 | case Time: 98 | v2, ok := v2.(Time) 99 | if !ok { 100 | return false 101 | } 102 | return time.Time(v1).Equal(time.Time(v2)) 103 | } 104 | return v1 == v2 105 | } 106 | 107 | // CompareValues return 0 if values are equal, positive value if first value sorts after second, and negative otherwise. 108 | func CompareValues(v1, v2 Value) int { 109 | switch v1 := v1.(type) { 110 | case Document: 111 | v2, ok := v2.(Document) 112 | if !ok { 113 | return -1 114 | } else if len(v1) != len(v2) { 115 | return len(v1) - len(v2) 116 | } 117 | return -1 // TODO: implement proper sorting? 118 | case Strings: 119 | v2, ok := v2.(Strings) 120 | if !ok { 121 | return -1 122 | } else if len(v1) != len(v2) { 123 | return len(v1) - len(v2) 124 | } 125 | for i := range v1 { 126 | if dn := CompareValues(String(v1[i]), String(v2[i])); dn != 0 { 127 | return dn 128 | } 129 | } 130 | return 0 131 | case Bytes: 132 | v2, ok := v2.(Bytes) 133 | if !ok { 134 | return -1 135 | } 136 | return bytes.Compare(v1, v2) 137 | case Time: 138 | v2, ok := v2.(Time) 139 | if !ok { 140 | return -1 141 | } 142 | t1, t2 := time.Time(v1), time.Time(v2) 143 | if t1.Equal(t2) { 144 | return 0 145 | } else if t1.Before(t2) { 146 | return -1 147 | } 148 | return +1 149 | case String: 150 | v2, ok := v2.(String) 151 | if !ok { 152 | return -1 153 | } 154 | return strings.Compare(string(v1), string(v2)) 155 | case Int: 156 | v2, ok := v2.(Int) 157 | if !ok { 158 | return -1 159 | } 160 | if v1 == v2 { 161 | return 0 162 | } else if v1 < v2 { 163 | return -1 164 | } 165 | return +1 166 | case Float: 167 | v2, ok := v2.(Float) 168 | if !ok { 169 | return -1 170 | } 171 | if v1 == v2 { 172 | return 0 173 | } else if v1 < v2 { 174 | return -1 175 | } 176 | return +1 177 | } 178 | return -1 179 | } 180 | -------------------------------------------------------------------------------- /legacy/nosql/value_test.go: -------------------------------------------------------------------------------- 1 | package nosql 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | var filterMatch = []struct { 10 | f FieldFilter 11 | d Document 12 | exp bool 13 | }{ 14 | { 15 | f: FieldFilter{Path: []string{"value", "str"}, Filter: GT, Value: String("f")}, 16 | d: Document{"value": Document{"str": String("bob")}}, 17 | exp: false, 18 | }, 19 | { 20 | f: FieldFilter{Path: []string{"value", "str"}, Filter: Equal, Value: String("f")}, 21 | d: Document{"value": Document{"str": String("bob")}}, 22 | exp: false, 23 | }, 24 | { 25 | f: FieldFilter{Path: []string{"value", "str"}, Filter: Equal, Value: String("bob")}, 26 | d: Document{"value": Document{"str": String("bob")}}, 27 | exp: true, 28 | }, 29 | { 30 | f: FieldFilter{Path: []string{"value", "str"}, Filter: NotEqual, Value: String("bob")}, 31 | d: Document{"value1": Document{"str": String("bob")}}, 32 | exp: true, 33 | }, 34 | } 35 | 36 | func TestFilterMatch(t *testing.T) { 37 | for _, c := range filterMatch { 38 | require.Equal(t, c.exp, c.f.Matches(c.d)) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /tuple/datastore/datastore_test.go: -------------------------------------------------------------------------------- 1 | package datastore 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "cloud.google.com/go/datastore" 9 | "github.com/ory/dockertest" 10 | 11 | "github.com/hidal-go/hidalgo/tuple" 12 | "github.com/hidal-go/hidalgo/tuple/tupletest" 13 | ) 14 | 15 | func TestDatastore(t *testing.T) { 16 | tupletest.RunTest(t, func(t testing.TB) tuple.Store { 17 | pool, err := dockertest.NewPool("") 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | 22 | const ( 23 | proj = "test" 24 | ) 25 | 26 | cont, err := pool.RunWithOptions(&dockertest.RunOptions{ 27 | Repository: "singularities/datastore-emulator", 28 | Tag: "latest", 29 | Env: []string{ 30 | "DATASTORE_LISTEN_ADDRESS=0.0.0.0:8080", 31 | "DATASTORE_PROJECT_ID=" + proj, 32 | }, 33 | ExposedPorts: []string{ 34 | "8080/tcp", 35 | }, 36 | }) 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | t.Cleanup(func() { 41 | _ = cont.Close() 42 | }) 43 | 44 | ctx := context.Background() 45 | host := cont.GetHostPort("8080/tcp") 46 | if host == "" { 47 | t.Fatal("empty host") 48 | } 49 | if err = os.Setenv("DATASTORE_EMULATOR_HOST", host); err != nil { 50 | t.Fatal(err) 51 | } else if host := os.Getenv("DATASTORE_EMULATOR_HOST"); host == "" { 52 | t.Fatal("set env failed") 53 | } 54 | cli, err := datastore.NewClient(ctx, proj) 55 | if err != nil { 56 | t.Fatal(err) 57 | } 58 | t.Cleanup(func() { 59 | _ = cli.Close() 60 | }) 61 | return OpenClient(cli) 62 | }, nil) 63 | } 64 | -------------------------------------------------------------------------------- /tuple/helpers.go: -------------------------------------------------------------------------------- 1 | package tuple 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | // Update is a helper to open a read-write transaction and update the database. 8 | func Update(ctx context.Context, s Store, update func(tx Tx) error) error { 9 | tx, err := s.Tx(ctx, true) 10 | if err != nil { 11 | return err 12 | } 13 | if err = update(tx); err != nil { 14 | defer tx.Close() 15 | return err 16 | } 17 | if err = tx.Commit(ctx); err != nil { 18 | defer tx.Close() 19 | return err 20 | } 21 | return tx.Close() 22 | } 23 | 24 | // View is a helper to open a read-only transaction to read the database. 25 | func View(ctx context.Context, s Store, view func(tx Tx) error) error { 26 | tx, err := s.Tx(ctx, false) 27 | if err != nil { 28 | return err 29 | } 30 | defer tx.Close() 31 | if err = view(tx); err != nil { 32 | return err 33 | } 34 | return tx.Close() 35 | } 36 | 37 | type LazyTuple interface { 38 | Key() Key 39 | Data() Data 40 | } 41 | 42 | func FilterIterator(it LazyTuple, f *Filter, next func() bool) bool { 43 | if f.IsAny() { 44 | return next() 45 | } 46 | for next() { 47 | if f.KeyFilter != nil && !f.KeyFilter.FilterKey(it.Key()) { 48 | continue 49 | } 50 | if f.DataFilter != nil && !f.DataFilter.FilterData(it.Data()) { 51 | continue 52 | } 53 | return true 54 | } 55 | return false 56 | } 57 | 58 | type Deleter interface { 59 | // DeleteTuplesByKey removes tuples by key. 60 | DeleteTuplesByKey(ctx context.Context, keys []Key) error 61 | Scanner 62 | } 63 | 64 | func DeleteEach(ctx context.Context, d Deleter, f *Filter) error { 65 | // TODO: recognize fixed filters 66 | it := d.Scan(ctx, &ScanOptions{ 67 | KeysOnly: true, 68 | Filter: f, 69 | }) 70 | defer it.Close() 71 | 72 | const batch = 100 73 | var buf []Key 74 | flush := func() error { 75 | if len(buf) == 0 { 76 | return nil 77 | } 78 | err := d.DeleteTuplesByKey(ctx, buf) 79 | if err != nil { 80 | return err 81 | } 82 | buf = buf[:0] 83 | return nil 84 | } 85 | for it.Next(ctx) { 86 | buf = append(buf, it.Key()) 87 | if len(buf) >= batch { 88 | if err := flush(); err != nil { 89 | return err 90 | } 91 | } 92 | } 93 | if err := flush(); err != nil { 94 | return err 95 | } 96 | return it.Err() 97 | } 98 | -------------------------------------------------------------------------------- /tuple/kv/downgrade.go: -------------------------------------------------------------------------------- 1 | package tuplekv 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/hidal-go/hidalgo/filter" 8 | "github.com/hidal-go/hidalgo/kv/flat" 9 | "github.com/hidal-go/hidalgo/tuple" 10 | "github.com/hidal-go/hidalgo/values" 11 | ) 12 | 13 | func NewKV(ctx context.Context, db tuple.Store, table string) (flat.KV, error) { 14 | tx, err := db.Tx(ctx, true) 15 | if err != nil { 16 | return nil, err 17 | } 18 | defer tx.Close() 19 | _, err = tx.Table(ctx, table) 20 | if err == tuple.ErrTableNotFound { 21 | _, err = tx.CreateTable(ctx, tuple.Header{ 22 | Name: table, 23 | Key: []tuple.KeyField{ 24 | {Name: "key", Type: values.BytesType{}}, 25 | }, 26 | Data: []tuple.Field{ 27 | {Name: "val", Type: values.BytesType{}}, 28 | }, 29 | }) 30 | } 31 | if err == nil { 32 | err = tx.Commit(ctx) 33 | } 34 | if err != nil { 35 | return nil, err 36 | } 37 | return &flatKV{db: db, table: table}, nil 38 | } 39 | 40 | type flatKV struct { 41 | db tuple.Store 42 | table string 43 | } 44 | 45 | func (kv *flatKV) Close() error { 46 | return kv.db.Close() 47 | } 48 | 49 | func (kv *flatKV) Tx(ctx context.Context, rw bool) (flat.Tx, error) { 50 | tx, err := kv.db.Tx(ctx, rw) 51 | if err != nil { 52 | return nil, err 53 | } 54 | tbl, err := tx.Table(ctx, kv.table) 55 | if err != nil { 56 | tx.Close() 57 | return nil, err 58 | } 59 | return &flatTx{tx: tx, tbl: tbl}, nil 60 | } 61 | 62 | func (kv *flatKV) View(ctx context.Context, fn func(tx flat.Tx) error) error { 63 | return flat.View(ctx, kv, fn) 64 | } 65 | 66 | func (kv *flatKV) Update(ctx context.Context, fn func(tx flat.Tx) error) error { 67 | return flat.Update(ctx, kv, fn) 68 | } 69 | 70 | type flatTx struct { 71 | tx tuple.Tx 72 | tbl tuple.Table 73 | } 74 | 75 | func (tx *flatTx) Commit(ctx context.Context) error { 76 | return tx.tx.Commit(ctx) 77 | } 78 | 79 | func (tx *flatTx) Close() error { 80 | return tx.tx.Close() 81 | } 82 | 83 | func flatKeyPart(b flat.Key) values.Bytes { 84 | return values.Bytes(b.Clone()) 85 | } 86 | 87 | func flatKey(b flat.Key) tuple.Key { 88 | if b == nil { 89 | return nil 90 | } 91 | return tuple.Key{flatKeyPart(b)} 92 | } 93 | 94 | func flatData(b flat.Value) tuple.Data { 95 | return tuple.Data{values.Bytes(b.Clone())} 96 | } 97 | 98 | func (tx *flatTx) Get(ctx context.Context, key flat.Key) (flat.Value, error) { 99 | row, err := tx.tbl.GetTuple(ctx, flatKey(key)) 100 | if err == tuple.ErrNotFound { 101 | return nil, flat.ErrNotFound 102 | } else if err != nil { 103 | return nil, err 104 | } 105 | b, ok := row[0].(values.Bytes) 106 | if !ok || b == nil { 107 | return nil, fmt.Errorf("unexpected value type: %T", row[0]) 108 | } 109 | return flat.Value(b), nil 110 | } 111 | 112 | func (tx *flatTx) GetBatch(ctx context.Context, key []flat.Key) ([]flat.Value, error) { 113 | keys := make([]tuple.Key, 0, len(key)) 114 | for _, k := range key { 115 | keys = append(keys, flatKey(k)) 116 | } 117 | rows, err := tx.tbl.GetTupleBatch(ctx, keys) 118 | if err != nil { 119 | return nil, err 120 | } 121 | vals := make([]flat.Value, len(key)) 122 | for i, d := range rows { 123 | if d == nil { 124 | continue 125 | } 126 | b, ok := d[0].(values.Bytes) 127 | if !ok || b == nil { 128 | return nil, fmt.Errorf("unexpected value type: %T", d[0]) 129 | } 130 | vals[i] = flat.Value(b) 131 | } 132 | return vals, nil 133 | } 134 | 135 | func (tx *flatTx) Put(ctx context.Context, k flat.Key, v flat.Value) error { 136 | return tx.tbl.UpdateTuple(ctx, tuple.Tuple{ 137 | Key: flatKey(k), 138 | Data: flatData(v), 139 | }, &tuple.UpdateOpt{Upsert: true}) 140 | } 141 | 142 | func (tx *flatTx) Del(ctx context.Context, k flat.Key) error { 143 | return tx.tbl.DeleteTuples(ctx, &tuple.Filter{ 144 | KeyFilter: tuple.Keys{flatKey(k)}, 145 | }) 146 | } 147 | 148 | func (tx *flatTx) Scan(ctx context.Context, opts ...flat.IteratorOption) flat.Iterator { 149 | tit := &flatIterator{ctx: ctx, tx: tx} 150 | tit.seek(ctx, nil) 151 | var it flat.Iterator = tit 152 | it = flat.ApplyIteratorOptions(it, opts) 153 | return it 154 | } 155 | 156 | var ( 157 | _ flat.Seeker = &flatIterator{} 158 | _ flat.PrefixIterator = &flatIterator{} 159 | ) 160 | 161 | type flatIterator struct { 162 | ctx context.Context 163 | tx *flatTx 164 | pref flat.Key 165 | it tuple.Iterator 166 | err error 167 | } 168 | 169 | func (it *flatIterator) Reset() { 170 | it.err = nil 171 | if it.it != nil { 172 | it.it.Reset() 173 | } 174 | } 175 | 176 | func (it *flatIterator) WithPrefix(pref flat.Key) flat.Iterator { 177 | it.pref = pref 178 | it.seek(it.ctx, nil) 179 | return it 180 | } 181 | 182 | func (it *flatIterator) Close() error { 183 | return it.it.Close() 184 | } 185 | 186 | func (it *flatIterator) Err() error { 187 | if err := it.it.Err(); err != nil { 188 | return err 189 | } 190 | return it.err 191 | } 192 | 193 | func (it *flatIterator) filters() tuple.KeyFilters { 194 | if len(it.pref) == 0 { 195 | return nil 196 | } 197 | return tuple.KeyFilters{ 198 | filter.Prefix(flatKeyPart(it.pref)), 199 | } 200 | } 201 | 202 | func (it *flatIterator) seek(ctx context.Context, key flat.Key) { 203 | it.Reset() 204 | if it.it != nil { 205 | _ = it.it.Close() 206 | } 207 | filters := it.filters() 208 | if len(key) != 0 { 209 | filters = append(filters, filter.GTE(flatKeyPart(key))) 210 | } 211 | var f *tuple.Filter 212 | if len(filters) != 0 { 213 | f = &tuple.Filter{KeyFilter: filters} 214 | } 215 | it.it = it.tx.tbl.Scan(ctx, &tuple.ScanOptions{ 216 | Sort: tuple.SortAsc, 217 | Filter: f, 218 | }) 219 | } 220 | 221 | func (it *flatIterator) Seek(ctx context.Context, key flat.Key) bool { 222 | it.seek(ctx, key) 223 | return it.it.Next(ctx) 224 | } 225 | 226 | func (it *flatIterator) Next(ctx context.Context) bool { 227 | if it.err != nil { 228 | return false 229 | } 230 | return it.it.Next(ctx) 231 | } 232 | 233 | func (it *flatIterator) Key() flat.Key { 234 | key := it.it.Key() 235 | if len(key) == 0 { 236 | return nil 237 | } else if len(key) > 1 { 238 | it.err = fmt.Errorf("unexpected key size: %d", len(key)) 239 | return nil 240 | } 241 | b, ok := key[0].(values.Bytes) 242 | if !ok || b == nil { 243 | it.err = fmt.Errorf("unexpected key type: %T", key[0]) 244 | return nil 245 | } 246 | return flat.Key(b).Clone() 247 | } 248 | 249 | func (it *flatIterator) Val() flat.Value { 250 | data := it.it.Data() 251 | if len(data) == 0 { 252 | return nil 253 | } 254 | b, ok := data[0].(values.Bytes) 255 | if !ok || b == nil { 256 | it.err = fmt.Errorf("unexpected value type: %T", data[0]) 257 | return nil 258 | } 259 | return flat.Value(b).Clone() 260 | } 261 | -------------------------------------------------------------------------------- /tuple/kv/upgrade_test.go: -------------------------------------------------------------------------------- 1 | package tuplekv_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/hidal-go/hidalgo/kv/flat" 7 | "github.com/hidal-go/hidalgo/kv/flat/btree" 8 | "github.com/hidal-go/hidalgo/tuple" 9 | tuplekv "github.com/hidal-go/hidalgo/tuple/kv" 10 | "github.com/hidal-go/hidalgo/tuple/tupletest" 11 | ) 12 | 13 | func TestKV2Tuple(t *testing.T) { 14 | tupletest.RunTest(t, func(t testing.TB) tuple.Store { 15 | kdb := btree.New() 16 | db := tuplekv.New(flat.Upgrade(kdb)) 17 | return db 18 | }, &tupletest.Options{ 19 | NoLocks: true, 20 | }) 21 | } 22 | -------------------------------------------------------------------------------- /tuple/sql/all/all.go: -------------------------------------------------------------------------------- 1 | package all 2 | 3 | import ( 4 | _ "github.com/hidal-go/hidalgo/tuple/sql/mysql" 5 | _ "github.com/hidal-go/hidalgo/tuple/sql/postgres" 6 | ) 7 | -------------------------------------------------------------------------------- /tuple/sql/builder.go: -------------------------------------------------------------------------------- 1 | package sqltuple 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | ) 7 | 8 | func (d *Dialect) NewBuilder() *Builder { 9 | return &Builder{d: d, buf: bytes.NewBuffer(nil)} 10 | } 11 | 12 | type Builder struct { 13 | d *Dialect 14 | pi int 15 | buf *bytes.Buffer 16 | args []interface{} 17 | } 18 | 19 | func (b *Builder) Reset() { 20 | b.pi = 0 21 | b.buf.Reset() 22 | b.args = b.args[0:] 23 | } 24 | 25 | func (b *Builder) Write(s string) { 26 | b.buf.WriteString(s) 27 | } 28 | 29 | func (b *Builder) place(v interface{}) string { 30 | p := b.pi 31 | b.pi++ 32 | b.args = append(b.args, v) 33 | return b.d.Placeholder(p) 34 | } 35 | 36 | func (b *Builder) Place(args ...interface{}) { 37 | if len(args) == 1 { 38 | if _, ok := args[0].([]interface{}); ok { 39 | panic("forgot to expand arguments") 40 | } 41 | } 42 | arr := make([]string, 0, len(args)) 43 | for _, v := range args { 44 | arr = append(arr, b.place(v)) 45 | } 46 | b.Write(strings.Join(arr, ", ")) 47 | } 48 | 49 | func (b *Builder) Idents(names ...string) { 50 | arr := make([]string, 0, len(names)) 51 | for _, s := range names { 52 | arr = append(arr, b.d.QuoteIdentifier(s)) 53 | } 54 | b.Write(strings.Join(arr, ", ")) 55 | } 56 | 57 | func (b *Builder) Literal(s string) { 58 | b.Write(b.d.QuoteString(s)) 59 | } 60 | 61 | func (b *Builder) opPlace(names []string, op string, args []interface{}, sep string) { 62 | arr := make([]string, 0, len(names)) 63 | for i, name := range names { 64 | arr = append(arr, b.d.QuoteIdentifier(name)+` `+op+` `+b.place(args[i])) 65 | } 66 | b.Write(strings.Join(arr, sep)) 67 | } 68 | 69 | func (b *Builder) EqPlace(names []string, args []interface{}) { 70 | b.opPlace(names, "=", args, ", ") 71 | } 72 | 73 | func (b *Builder) EqPlaceAnd(names []string, args []interface{}) { 74 | b.Write("(") 75 | b.opPlace(names, "=", args, ") AND (") 76 | b.Write(")") 77 | } 78 | 79 | func (b *Builder) OpPlace(names []string, op string, args []interface{}) { 80 | b.opPlace(names, op, args, ", ") 81 | } 82 | 83 | func (b *Builder) OpPlaceAnd(names []string, op string, args []interface{}) { 84 | b.Write("(") 85 | b.opPlace(names, op, args, ") AND (") 86 | b.Write(")") 87 | } 88 | 89 | func (b *Builder) String() string { 90 | return b.buf.String() 91 | } 92 | 93 | func (b *Builder) Args() []interface{} { 94 | return b.args 95 | } 96 | -------------------------------------------------------------------------------- /tuple/sql/dialect.go: -------------------------------------------------------------------------------- 1 | package sqltuple 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/hidal-go/hidalgo/values" 8 | ) 9 | 10 | type ErrorFunc func(err error) error 11 | 12 | type Dialect struct { 13 | Errors ErrorFunc 14 | BytesType string 15 | StringType string 16 | BytesKeyType string 17 | StringKeyType string 18 | TimeType string 19 | StringTypeCollation string 20 | AutoType string 21 | QuoteIdentifierFunc func(s string) string 22 | Placeholder func(i int) string 23 | // DefaultSchema will be used to query table metadata. 24 | // If not set, defaults to the database name. 25 | DefaultSchema string 26 | // ListColumns is a query that will be executed to get columns info. 27 | // Two parameters will be passed to the query: current schema and the table name. 28 | ListColumns string 29 | // Unsigned indicates that a database supports UNSIGNED modifier for integer types. 30 | Unsigned bool 31 | // NoIteratorsWhenMutating mark indicates that backend cannot run iterators and 32 | // mutations in the same transaction (example: SELECT and DELETE while iterating). 33 | NoIteratorsWhenMutating bool 34 | // ReplaceStmt indicates that backend supports REPLACE statement. 35 | ReplaceStmt bool 36 | // OnConflict indicates that backend supports ON CONFLICT in INSERT statement. 37 | OnConflict bool 38 | // Returning indicates that INSERT queries needs an RETURNING keyword to return last 39 | // inserted id. 40 | Returning bool 41 | ColumnCommentInline func(s string) string 42 | ColumnCommentSet func(b *Builder, tbl, col, s string) 43 | } 44 | 45 | func (d *Dialect) SetDefaults() { 46 | if d.StringType == "" { 47 | d.StringType = "TEXT" 48 | } 49 | if d.StringKeyType == "" { 50 | d.StringKeyType = d.StringType 51 | } 52 | if d.BytesType == "" { 53 | d.BytesType = "BLOB" 54 | } 55 | if d.BytesKeyType == "" { 56 | d.BytesKeyType = d.BytesType 57 | } 58 | if d.TimeType == "" { 59 | d.TimeType = "TIMESTAMP" 60 | } 61 | if d.Placeholder == nil { 62 | d.Placeholder = func(_ int) string { 63 | return "?" 64 | } 65 | } 66 | } 67 | 68 | func (d *Dialect) QuoteIdentifier(s string) string { 69 | if q := d.QuoteIdentifierFunc; q != nil { 70 | return q(s) 71 | } 72 | return "`" + strings.Replace(s, "`", "", -1) + "`" 73 | } 74 | 75 | func (d *Dialect) QuoteString(s string) string { 76 | // only used when setting comments, so it's pretty naive 77 | return "'" + strings.Replace(s, "'", "''", -1) + "'" 78 | } 79 | 80 | func (d *Dialect) sqlType(t values.Type, key bool) string { 81 | var tp string 82 | switch t.(type) { 83 | case values.StringType: 84 | tp = d.StringType 85 | if key { 86 | tp = d.StringKeyType 87 | } 88 | if d.StringTypeCollation != "" { 89 | // TODO: set it on the table/database 90 | tp += " " + d.StringTypeCollation 91 | } 92 | case values.BytesType: 93 | tp = d.BytesType 94 | if key { 95 | tp = d.BytesKeyType 96 | } 97 | case values.IntType: 98 | tp = "BIGINT" 99 | case values.UIntType: 100 | tp = "BIGINT" 101 | if d.Unsigned { 102 | tp += " UNSIGNED" 103 | } 104 | case values.FloatType: 105 | tp = "DOUBLE PRECISION" 106 | case values.BoolType: 107 | tp = "BOOLEAN" 108 | case values.TimeType: 109 | tp = d.TimeType 110 | default: 111 | panic(fmt.Errorf("unsupported type: %T", t)) 112 | } 113 | if key { 114 | tp += " NOT NULL" 115 | } else { 116 | tp += " NULL" 117 | } 118 | return tp 119 | } 120 | 121 | func (d *Dialect) sqlColumnComment(t values.Type) string { 122 | var c string 123 | switch t.(type) { 124 | case values.BoolType: 125 | c = "Bool" 126 | case values.UIntType: 127 | if !d.Unsigned { 128 | c = "UInt" 129 | } 130 | } 131 | return c 132 | } 133 | 134 | func (d *Dialect) sqlColumnCommentAuto() string { 135 | return d.sqlColumnComment(values.UIntType{}) + " Auto" 136 | } 137 | 138 | func (d *Dialect) sqlColumnCommentInline(t values.Type) string { 139 | if d.ColumnCommentInline == nil { 140 | return "" 141 | } 142 | c := d.sqlColumnComment(t) 143 | if c == "" { 144 | return "" 145 | } 146 | return " " + d.ColumnCommentInline(d.QuoteString(c)) 147 | } 148 | 149 | func (d *Dialect) sqlColumnCommentAutoInline() string { 150 | if d.ColumnCommentInline == nil { 151 | return "" 152 | } 153 | c := d.sqlColumnCommentAuto() 154 | return " " + d.ColumnCommentInline(d.QuoteString(c)) 155 | } 156 | 157 | func (d *Dialect) nativeType(typ, comment string) (values.Type, bool, error) { 158 | typ = strings.ToLower(typ) 159 | auto := strings.HasSuffix(comment, " Auto") 160 | if auto { 161 | comment = comment[:len(comment)-5] 162 | } 163 | var opt string 164 | if i := strings.Index(typ, "("); i > 0 { 165 | typ, opt = typ[:i], typ[i:] 166 | } 167 | if i := strings.Index(typ, " "); i > 0 { 168 | typ, opt = typ[:i], typ[i:]+opt 169 | } 170 | switch typ { 171 | case "text", "varchar", "char": 172 | return values.StringType{}, auto, nil 173 | case "blob", "bytea", "varbinary", "binary": 174 | return values.BytesType{}, auto, nil 175 | case "double", "double precision": 176 | return values.FloatType{}, auto, nil 177 | case "boolean": 178 | return values.BoolType{}, auto, nil 179 | case "tinyint": 180 | if opt == "(1)" && comment == "Bool" { // TODO: or rather if it's MySQL 181 | return values.BoolType{}, auto, nil 182 | } 183 | fallthrough 184 | case "bigint", "int", "integer", "mediumint", "smallint": 185 | if strings.HasSuffix(opt, "unsigned") || comment == "UInt" { 186 | return values.UIntType{}, auto, nil 187 | } 188 | return values.IntType{}, auto, nil 189 | case "timestamp", "datetime", "date", "time": 190 | return values.TimeType{}, auto, nil 191 | } 192 | return nil, false, fmt.Errorf("unsupported column type: %q", typ) 193 | } 194 | -------------------------------------------------------------------------------- /tuple/sql/mysql/mysql.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/go-sql-driver/mysql" // This import will be dropped if mysql.MySQLError is removed. 7 | _ "github.com/go-sql-driver/mysql" // This side-effect import must be kept. 8 | 9 | "github.com/hidal-go/hidalgo/base" 10 | sqltuple "github.com/hidal-go/hidalgo/tuple/sql" 11 | ) 12 | 13 | const Name = "mysql" 14 | 15 | func init() { 16 | sqltuple.Register(sqltuple.Registration{ 17 | Registration: base.Registration{ 18 | Name: Name, Title: "MySQL", 19 | Local: false, Volatile: false, 20 | }, 21 | Driver: "mysql", 22 | DSN: func(addr, ns string) (string, error) { 23 | return addr + "/" + ns + "?parseTime=true", nil 24 | }, 25 | Dialect: sqltuple.Dialect{ 26 | StringType: "TEXT", 27 | BytesType: "BLOB", 28 | // TODO: pick size based on the number of columns (max 3k) 29 | StringKeyType: "VARCHAR(256)", 30 | BytesKeyType: "VARBINARY(256)", 31 | TimeType: "DATETIME(6)", 32 | // TODO: set it on the table/database 33 | StringTypeCollation: " CHARACTER SET utf8 COLLATE utf8_unicode_ci", 34 | Unsigned: true, 35 | ReplaceStmt: true, 36 | NoIteratorsWhenMutating: true, 37 | ListColumns: `SELECT column_name, column_type, is_nullable, column_key, column_comment 38 | FROM information_schema.columns WHERE table_schema = ? AND table_name = ?`, 39 | QuoteIdentifierFunc: func(s string) string { 40 | return "`" + strings.Replace(s, "`", "", -1) + "`" 41 | }, 42 | ColumnCommentInline: func(s string) string { 43 | return "COMMENT " + s 44 | }, 45 | Errors: func(err error) error { 46 | if e, ok := err.(*mysql.MySQLError); ok { 47 | switch e.Number { 48 | case 1146: 49 | return sqltuple.ErrTableNotFound 50 | } 51 | } 52 | return err 53 | }, 54 | }, 55 | }) 56 | } 57 | -------------------------------------------------------------------------------- /tuple/sql/mysql/mysql_test.go: -------------------------------------------------------------------------------- 1 | package mysql_test 2 | 3 | import ( 4 | "testing" 5 | 6 | _ "github.com/hidal-go/hidalgo/tuple/sql/mysql/test" 7 | 8 | "github.com/hidal-go/hidalgo/tuple/sql/mysql" 9 | "github.com/hidal-go/hidalgo/tuple/sql/sqltest" 10 | ) 11 | 12 | func TestMySQL(t *testing.T) { 13 | sqltest.Test(t, mysql.Name) 14 | } 15 | 16 | func BenchmarkMySQL(b *testing.B) { 17 | sqltest.Benchmark(b, mysql.Name) 18 | } 19 | -------------------------------------------------------------------------------- /tuple/sql/mysql/test/mysqltest.go: -------------------------------------------------------------------------------- 1 | package mysqltest 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ory/dockertest" 7 | 8 | sqltuple "github.com/hidal-go/hidalgo/tuple/sql" 9 | "github.com/hidal-go/hidalgo/tuple/sql/mysql" 10 | "github.com/hidal-go/hidalgo/tuple/sql/sqltest" 11 | ) 12 | 13 | var versions = []string{ 14 | "5.7", 15 | } 16 | 17 | func init() { 18 | var vers []sqltest.Version 19 | for _, v := range versions { 20 | vers = append(vers, sqltest.Version{ 21 | Name: v, Factory: MySQLVersion(v), 22 | }) 23 | } 24 | sqltest.Register(mysql.Name, vers...) 25 | } 26 | 27 | func MySQLVersion(vers string) sqltest.Database { 28 | const image = "mysql" 29 | return sqltest.Database{ 30 | Recreate: false, 31 | Run: func(t testing.TB) string { 32 | pool, err := dockertest.NewPool("") 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | 37 | cont, err := pool.Run(image, vers, []string{ 38 | "MYSQL_ROOT_PASSWORD=root", 39 | }) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | t.Cleanup(func() { 44 | _ = cont.Close() 45 | }) 46 | 47 | const port = "3306/tcp" 48 | addr := "root:root@(" + cont.GetHostPort(port) + ")" 49 | 50 | err = pool.Retry(func() error { 51 | cli, err := sqltuple.OpenSQL(mysql.Name, addr, "") 52 | if err != nil { 53 | return err 54 | } 55 | defer cli.Close() 56 | return cli.Ping() 57 | }) 58 | if err != nil { 59 | t.Fatal(err) 60 | } 61 | return addr 62 | }, 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /tuple/sql/postgres/postgres.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "strconv" 5 | 6 | "github.com/lib/pq" // This import will be dropped if pq.QuoteIdentifier is removed. 7 | _ "github.com/lib/pq" // This side-effect import must be kept. 8 | 9 | "github.com/hidal-go/hidalgo/base" 10 | sqltuple "github.com/hidal-go/hidalgo/tuple/sql" 11 | ) 12 | 13 | const Name = "postgres" 14 | 15 | func init() { 16 | sqltuple.Register(sqltuple.Registration{ 17 | Registration: base.Registration{ 18 | Name: Name, Title: "PostgreSQL", 19 | Local: false, Volatile: false, 20 | }, 21 | Driver: "postgres", 22 | DSN: func(addr, ns string) (string, error) { 23 | return addr + "/" + ns + "?sslmode=disable", nil 24 | }, 25 | Dialect: sqltuple.Dialect{ 26 | BytesType: "BYTEA", 27 | AutoType: "BIGSERIAL", 28 | QuoteIdentifierFunc: pq.QuoteIdentifier, 29 | DefaultSchema: "public", 30 | Unsigned: false, 31 | Returning: true, 32 | OnConflict: true, 33 | NoIteratorsWhenMutating: true, 34 | Placeholder: func(i int) string { 35 | return "$" + strconv.Itoa(i+1) 36 | }, 37 | Errors: func(err error) error { 38 | return err 39 | }, 40 | ListColumns: `SELECT c.column_name, c.data_type, c.is_nullable, tc.constraint_type, col_description(a.attrelid, a.attnum) 41 | FROM information_schema.columns c 42 | LEFT JOIN information_schema.constraint_column_usage AS ccu ON c.table_schema = ccu.table_schema AND c.table_name = ccu.table_name AND c.column_name = ccu.column_name 43 | LEFT JOIN information_schema.table_constraints AS tc ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND tc.constraint_name = ccu.constraint_name 44 | LEFT JOIN pg_catalog.pg_attribute AS a ON a.attname = c.column_name 45 | LEFT JOIN pg_catalog.pg_class AS pc ON a.attrelid = pc.oid AND pc.relname = c.table_name 46 | WHERE c.table_schema = $1 47 | AND c.table_name = $2 48 | AND a.attnum > 0 49 | AND a.attisdropped is false 50 | AND pg_catalog.pg_table_is_visible(pc.oid)`, 51 | ColumnCommentSet: func(b *sqltuple.Builder, tbl, col, s string) { 52 | b.Write(`COMMENT ON COLUMN `) 53 | b.Idents(tbl) 54 | b.Write(".") 55 | b.Idents(col) 56 | b.Write(` IS `) 57 | b.Literal(s) 58 | }, 59 | }, 60 | }) 61 | } 62 | -------------------------------------------------------------------------------- /tuple/sql/postgres/postgres_test.go: -------------------------------------------------------------------------------- 1 | package postgres_test 2 | 3 | import ( 4 | "testing" 5 | 6 | _ "github.com/hidal-go/hidalgo/tuple/sql/postgres/test" 7 | 8 | "github.com/hidal-go/hidalgo/tuple/sql/postgres" 9 | "github.com/hidal-go/hidalgo/tuple/sql/sqltest" 10 | ) 11 | 12 | func TestPostgreSQL(t *testing.T) { 13 | sqltest.Test(t, postgres.Name) 14 | } 15 | 16 | func BenchmarkPostgreSQL(b *testing.B) { 17 | sqltest.Benchmark(b, postgres.Name) 18 | } 19 | -------------------------------------------------------------------------------- /tuple/sql/postgres/test/postgrestest.go: -------------------------------------------------------------------------------- 1 | package mysqltest 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ory/dockertest" 7 | 8 | sqltuple "github.com/hidal-go/hidalgo/tuple/sql" 9 | "github.com/hidal-go/hidalgo/tuple/sql/postgres" 10 | "github.com/hidal-go/hidalgo/tuple/sql/sqltest" 11 | ) 12 | 13 | var versions = []string{ 14 | "13", 15 | } 16 | 17 | func init() { 18 | var vers []sqltest.Version 19 | for _, v := range versions { 20 | vers = append(vers, sqltest.Version{ 21 | Name: v, Factory: PostgresVersion(v), 22 | }) 23 | } 24 | sqltest.Register(postgres.Name, vers...) 25 | } 26 | 27 | func PostgresVersion(vers string) sqltest.Database { 28 | const image = "postgres" 29 | return sqltest.Database{ 30 | Recreate: false, 31 | Run: func(t testing.TB) string { 32 | pool, err := dockertest.NewPool("") 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | 37 | cont, err := pool.Run(image, vers, []string{ 38 | "POSTGRES_PASSWORD=postgres", 39 | }) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | t.Cleanup(func() { 44 | _ = cont.Close() 45 | }) 46 | 47 | const port = "5432/tcp" 48 | addr := `postgres://postgres:postgres@` + cont.GetHostPort(port) 49 | 50 | err = pool.Retry(func() error { 51 | cli, err := sqltuple.OpenSQL(postgres.Name, addr, "") 52 | if err != nil { 53 | return err 54 | } 55 | defer cli.Close() 56 | return cli.Ping() 57 | }) 58 | if err != nil { 59 | t.Fatal(err) 60 | } 61 | return addr 62 | }, 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /tuple/sql/registry.go: -------------------------------------------------------------------------------- 1 | package sqltuple 2 | 3 | import ( 4 | "sort" 5 | 6 | "github.com/hidal-go/hidalgo/base" 7 | ) 8 | 9 | // DSNFunc is a function for building a Data Source Name for SQL driver, given a address and the database name. 10 | type DSNFunc func(addr, ns string) (string, error) 11 | 12 | // Registration is an information about the database driver. 13 | type Registration struct { 14 | base.Registration 15 | Driver string 16 | DSN DSNFunc 17 | Dialect Dialect 18 | } 19 | 20 | var registry = make(map[string]Registration) 21 | 22 | // Register globally registers a database driver. 23 | func Register(reg Registration) { 24 | if reg.Name == "" { 25 | panic("name cannot be empty") 26 | } else if _, ok := registry[reg.Name]; ok { 27 | panic(base.ErrRegistered{Name: reg.Name}) 28 | } 29 | registry[reg.Name] = reg 30 | } 31 | 32 | // List enumerates all globally registered database drivers. 33 | func List() []Registration { 34 | out := make([]Registration, 0, len(registry)) 35 | for _, r := range registry { 36 | out = append(out, r) 37 | } 38 | sort.Slice(out, func(i, j int) bool { 39 | return out[i].Name < out[j].Name 40 | }) 41 | return out 42 | } 43 | 44 | // ByName returns a registered database driver by it's name. 45 | func ByName(name string) *Registration { 46 | r, ok := registry[name] 47 | if !ok { 48 | return nil 49 | } 50 | return &r 51 | } 52 | -------------------------------------------------------------------------------- /tuple/sql/sqltest/all/alltest.go: -------------------------------------------------------------------------------- 1 | package alltest 2 | 3 | import ( 4 | _ "github.com/hidal-go/hidalgo/tuple/sql/mysql/test" 5 | _ "github.com/hidal-go/hidalgo/tuple/sql/postgres/test" 6 | ) 7 | -------------------------------------------------------------------------------- /tuple/sql/sqltest/registry.go: -------------------------------------------------------------------------------- 1 | package sqltest 2 | 3 | import ( 4 | "sort" 5 | "testing" 6 | 7 | sqltuple "github.com/hidal-go/hidalgo/tuple/sql" 8 | ) 9 | 10 | type Registration struct { 11 | sqltuple.Registration 12 | Versions []Version 13 | } 14 | 15 | type Version struct { 16 | Name string 17 | Factory Database 18 | } 19 | 20 | var registry = make(map[string][]Version) 21 | 22 | // Register globally registers a database driver. 23 | func Register(name string, vers ...Version) { 24 | if name == "" { 25 | panic("name cannot be empty") 26 | } else if len(vers) == 0 { 27 | panic("at least one version should be specified") 28 | } else if r := sqltuple.ByName(name); r == nil { 29 | panic("name is not registered") 30 | } 31 | vers = append([]Version{}, vers...) 32 | sort.Slice(vers, func(i, j int) bool { 33 | return vers[i].Name < vers[j].Name 34 | }) 35 | registry[name] = vers 36 | } 37 | 38 | // List enumerates all globally registered database drivers. 39 | func List() []Registration { 40 | out := make([]Registration, 0, len(registry)) 41 | for name, vers := range registry { 42 | out = append(out, Registration{ 43 | Registration: *sqltuple.ByName(name), 44 | Versions: append([]Version{}, vers...), 45 | }) 46 | } 47 | sort.Slice(out, func(i, j int) bool { 48 | return out[i].Name < out[j].Name 49 | }) 50 | return out 51 | } 52 | 53 | // ByName returns a registered database driver by it's name. 54 | func ByName(name string) *Registration { 55 | vers, ok := registry[name] 56 | if !ok { 57 | return nil 58 | } 59 | return &Registration{ 60 | Registration: *sqltuple.ByName(name), 61 | Versions: append([]Version{}, vers...), 62 | } 63 | } 64 | 65 | func allNames() []string { 66 | var names []string 67 | for name := range registry { 68 | names = append(names, name) 69 | } 70 | return names 71 | } 72 | 73 | func runT(t *testing.T, test func(t *testing.T, name string, run Database), name string) { 74 | for _, v := range ByName(name).Versions { 75 | t.Run(v.Name, func(tt *testing.T) { test(tt, name, v.Factory) }) 76 | } 77 | } 78 | 79 | func runB(b *testing.B, bench func(b *testing.B, run Database), name string) { 80 | for _, v := range ByName(name).Versions { 81 | b.Run(v.Name, func(bb *testing.B) { bench(bb, v.Factory) }) 82 | } 83 | } 84 | 85 | func RunTest(t *testing.T, test func(t *testing.T, name string, run Database), names ...string) { 86 | for _, name := range names { 87 | if _, ok := registry[name]; !ok { 88 | panic("not registered: " + name) 89 | } 90 | } 91 | if len(names) == 0 { 92 | names = allNames() 93 | } 94 | for _, name := range names { 95 | t.Run(name, func(tt *testing.T) { runT(tt, test, name) }) 96 | } 97 | } 98 | 99 | func RunBenchmark(b *testing.B, bench func(b *testing.B, run Database), names ...string) { 100 | for _, name := range names { 101 | if _, ok := registry[name]; !ok { 102 | panic("not registered: " + name) 103 | } 104 | } 105 | if len(names) == 0 { 106 | names = allNames() 107 | } 108 | for _, name := range names { 109 | b.Run(name, func(bb *testing.B) { runB(bb, bench, name) }) 110 | } 111 | } 112 | 113 | func Test(t *testing.T, names ...string) { 114 | RunTest(t, TestSQL, names...) 115 | } 116 | 117 | func Benchmark(b *testing.B, names ...string) { 118 | RunBenchmark(b, BenchmarkSQL, names...) 119 | } 120 | -------------------------------------------------------------------------------- /tuple/sql/sqltest/sqltest.go: -------------------------------------------------------------------------------- 1 | package sqltest 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/hidal-go/hidalgo/tuple" 12 | sqltuple "github.com/hidal-go/hidalgo/tuple/sql" 13 | "github.com/hidal-go/hidalgo/tuple/tupletest" 14 | ) 15 | 16 | type Database struct { 17 | Recreate bool 18 | Run func(t testing.TB) string 19 | } 20 | 21 | func TestSQL(t *testing.T, name string, gen Database) { 22 | var addr string 23 | recreate := gen.Recreate 24 | if !recreate { 25 | addr = gen.Run(t) 26 | } 27 | tupletest.RunTest(t, func(t testing.TB) tuple.Store { 28 | db := fmt.Sprintf("db_%x", rand.Int()) 29 | addr := addr 30 | if recreate { 31 | addr = gen.Run(t) 32 | } 33 | conn, err := sqltuple.OpenSQL(name, addr, "") 34 | if err != nil { 35 | require.NoError(t, err) 36 | } 37 | _, err = conn.Exec(`CREATE DATABASE ` + db) 38 | conn.Close() 39 | if err != nil { 40 | require.NoError(t, err) 41 | } 42 | conn, err = sqltuple.OpenSQL(name, addr, db) 43 | if err != nil { 44 | require.NoError(t, err) 45 | } 46 | t.Cleanup(func() { 47 | conn.Close() 48 | if !recreate { 49 | conn, err := sqltuple.OpenSQL(name, addr, "") 50 | if err == nil { 51 | _, err = conn.Exec(`DROP DATABASE ` + db) 52 | conn.Close() 53 | } 54 | if err != nil { 55 | t.Errorf("cannot remove test database: %v", err) 56 | } 57 | } 58 | }) 59 | return sqltuple.New(conn, db, sqltuple.ByName(name).Dialect) 60 | }, nil) 61 | } 62 | 63 | func BenchmarkSQL(t *testing.B, gen Database) { 64 | // TODO 65 | } 66 | 67 | func init() { 68 | rand.Seed(time.Now().UnixNano()) 69 | } 70 | -------------------------------------------------------------------------------- /tuple/stats.go: -------------------------------------------------------------------------------- 1 | package tuple 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | ) 7 | 8 | // ErrWildGuess returned if the the size can only be randomly guessed by the backend without scanning the data. 9 | var ErrWildGuess = errors.New("can only guess the size") 10 | 11 | // TableSize returns a number of records in a table matching the filter. 12 | // If exact is set to false, an estimate will be returned. 13 | // If estimate cannot be obtained without scanning the whole table, ErrWildGuess will be returned 14 | // with some random number. 15 | func TableSize(ctx context.Context, t Table, f *Filter, exact bool) (int64, error) { 16 | // TODO: optimize for backends that can provide this functionality 17 | if !exact { 18 | return 1000, ErrWildGuess 19 | } 20 | it := t.Scan(ctx, &ScanOptions{ 21 | KeysOnly: true, 22 | Filter: f, 23 | }) 24 | var n int64 25 | for it.Next(ctx) { 26 | n++ 27 | } 28 | return n, it.Err() 29 | } 30 | -------------------------------------------------------------------------------- /tuple/tuplepb/tuple.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package nwca.hidalgo.tuple; 4 | 5 | option go_package = "tuplepb"; 6 | 7 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 8 | option (gogoproto.protosizer_all) = true; 9 | option (gogoproto.marshaler_all) = true; 10 | option (gogoproto.unmarshaler_all) = true; 11 | 12 | enum ValueType { 13 | TYPE_ANY = 0; 14 | TYPE_BYTES = 1; 15 | TYPE_STRING = 2; 16 | TYPE_UINT = 3; 17 | TYPE_INT = 4; 18 | TYPE_BOOL = 5; 19 | TYPE_TIME = 6; 20 | TYPE_FLOAT = 7; 21 | } 22 | 23 | message Table { 24 | string name = 1; 25 | repeated KeyField key = 2 [(gogoproto.nullable) = false]; 26 | repeated Field data = 3 [(gogoproto.nullable) = false]; 27 | } 28 | 29 | message KeyField { 30 | string name = 1; 31 | ValueType type = 2; 32 | bool auto = 3; 33 | } 34 | 35 | message Field { 36 | string name = 1; 37 | ValueType type = 2; 38 | } 39 | -------------------------------------------------------------------------------- /tuple/tuplepb/tuplepb.go: -------------------------------------------------------------------------------- 1 | package tuplepb 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/hidal-go/hidalgo/tuple" 7 | "github.com/hidal-go/hidalgo/values" 8 | ) 9 | 10 | //go:generate protoc --proto_path=$GOPATH/src:. --gogo_out=. tuple.proto 11 | 12 | // Make sure that new fields will cause compilation error. 13 | var ( 14 | _ tuple.Header = struct { 15 | Name string 16 | Key []tuple.KeyField 17 | Data []tuple.Field 18 | }{} 19 | 20 | _ tuple.KeyField = struct { 21 | Name string 22 | Type values.SortableType 23 | Auto bool 24 | }{} 25 | 26 | _ tuple.Field = struct { 27 | Name string 28 | Type values.Type 29 | }{} 30 | ) 31 | 32 | var ( 33 | value2type = make(map[values.Type]ValueType) 34 | type2sortable = make(map[ValueType]values.SortableType) 35 | type2value = map[ValueType]values.Type{ 36 | ValueType_TYPE_ANY: nil, 37 | ValueType_TYPE_BYTES: values.BytesType{}, 38 | ValueType_TYPE_STRING: values.StringType{}, 39 | ValueType_TYPE_UINT: values.UIntType{}, 40 | ValueType_TYPE_INT: values.IntType{}, 41 | ValueType_TYPE_BOOL: values.BoolType{}, 42 | ValueType_TYPE_TIME: values.TimeType{}, 43 | ValueType_TYPE_FLOAT: values.FloatType{}, 44 | } 45 | ) 46 | 47 | func init() { 48 | for typ, v := range type2value { 49 | if _, ok := value2type[v]; ok { 50 | panic(typ.String()) 51 | } 52 | value2type[v] = typ 53 | if v, ok := v.(values.SortableType); ok && v != nil { 54 | type2sortable[typ] = v 55 | } 56 | } 57 | } 58 | 59 | func typeOf(v values.Type) (ValueType, bool) { 60 | typ, ok := value2type[v] 61 | return typ, ok 62 | } 63 | 64 | func MarshalTable(t *tuple.Header) ([]byte, error) { 65 | table := Table{ 66 | Name: t.Name, 67 | Key: make([]KeyField, 0, len(t.Key)), 68 | Data: make([]Field, 0, len(t.Data)), 69 | } 70 | for _, f := range t.Key { 71 | tp, ok := typeOf(f.Type) 72 | if !ok { 73 | return nil, fmt.Errorf("unsupported key type: %T", f.Type) 74 | } 75 | table.Key = append(table.Key, KeyField{ 76 | Name: f.Name, Type: tp, Auto: f.Auto, 77 | }) 78 | } 79 | for _, f := range t.Data { 80 | tp, ok := typeOf(f.Type) 81 | if !ok { 82 | return nil, fmt.Errorf("unsupported value type: %T", f.Type) 83 | } 84 | table.Data = append(table.Data, Field{ 85 | Name: f.Name, Type: tp, 86 | }) 87 | } 88 | return table.Marshal() 89 | } 90 | 91 | func UnmarshalTable(p []byte) (*tuple.Header, error) { 92 | var t Table 93 | if err := t.Unmarshal(p); err != nil { 94 | return nil, err 95 | } 96 | table := &tuple.Header{ 97 | Name: t.Name, 98 | Key: make([]tuple.KeyField, 0, len(t.Key)), 99 | Data: make([]tuple.Field, 0, len(t.Data)), 100 | } 101 | 102 | for _, f := range t.Key { 103 | tp, ok := type2sortable[f.Type] 104 | if !ok { 105 | return nil, fmt.Errorf("unsupported key type: %T", f.Type) 106 | } 107 | table.Key = append(table.Key, tuple.KeyField{ 108 | Name: f.Name, Type: tp, Auto: f.Auto, 109 | }) 110 | } 111 | for _, f := range t.Data { 112 | tp, ok := type2value[f.Type] 113 | if !ok { 114 | return nil, fmt.Errorf("unsupported value type: %T", f.Type) 115 | } 116 | table.Data = append(table.Data, tuple.Field{ 117 | Name: f.Name, Type: tp, 118 | }) 119 | } 120 | return table, nil 121 | } 122 | -------------------------------------------------------------------------------- /tuple/tuplepb/tuplepb_test.go: -------------------------------------------------------------------------------- 1 | package tuplepb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/hidal-go/hidalgo/tuple" 9 | "github.com/hidal-go/hidalgo/values" 10 | ) 11 | 12 | func TestTableEncoding(t *testing.T) { 13 | tbl := &tuple.Header{ 14 | Name: "test", 15 | Key: []tuple.KeyField{ 16 | {Name: "k1", Type: values.IntType{}, Auto: true}, 17 | {Name: "k2", Type: values.StringType{}}, 18 | }, 19 | Data: []tuple.Field{ 20 | {Name: "f1", Type: values.BytesType{}}, 21 | {Name: "f2", Type: values.StringType{}}, 22 | {Name: "f2", Type: values.FloatType{}}, 23 | }, 24 | } 25 | 26 | data, err := MarshalTable(tbl) 27 | require.NoError(t, err) 28 | 29 | tbl2, err := UnmarshalTable(data) 30 | require.NoError(t, err) 31 | require.Equal(t, tbl, tbl2) 32 | } 33 | -------------------------------------------------------------------------------- /tuple/tupletest/helpers.go: -------------------------------------------------------------------------------- 1 | package tupletest 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/hidal-go/hidalgo/tuple" 7 | ) 8 | 9 | func NewTest(t testing.TB, db tuple.Store) *Test { 10 | return &Test{t: t, db: db} 11 | } 12 | 13 | type Test struct { 14 | t testing.TB 15 | db tuple.Store 16 | } 17 | -------------------------------------------------------------------------------- /tuple/tupletest/tupletest.go: -------------------------------------------------------------------------------- 1 | package tupletest 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | 12 | "github.com/hidal-go/hidalgo/filter" 13 | hkv "github.com/hidal-go/hidalgo/kv" 14 | "github.com/hidal-go/hidalgo/kv/flat" 15 | "github.com/hidal-go/hidalgo/kv/kvtest" 16 | "github.com/hidal-go/hidalgo/tuple" 17 | tuplekv "github.com/hidal-go/hidalgo/tuple/kv" 18 | "github.com/hidal-go/hidalgo/values" 19 | ) 20 | 21 | // Func is a constructor for database implementations. 22 | // It returns an empty database and a function to destroy it. 23 | type Func func(t testing.TB) tuple.Store 24 | 25 | type Options struct { 26 | NoLocks bool // not safe for concurrent writes 27 | } 28 | 29 | // RunTest runs all tests for tuple store implementations. 30 | func RunTest(t *testing.T, fnc Func, opts *Options) { 31 | if opts == nil { 32 | opts = &Options{} 33 | } 34 | for _, c := range testList { 35 | t.Run(c.name, func(t *testing.T) { 36 | db := fnc(t) 37 | c.test(t, db) 38 | }) 39 | } 40 | t.Run("kv", func(t *testing.T) { 41 | kvtest.RunTest(t, func(t testing.TB) hkv.KV { 42 | db := fnc(t) 43 | 44 | ctx := context.Background() 45 | kdb, err := tuplekv.NewKV(ctx, db, "kv") 46 | if err != nil { 47 | require.NoError(t, err) 48 | } 49 | t.Cleanup(func() { 50 | _ = kdb.Close() 51 | }) 52 | return flat.Upgrade(kdb) 53 | }, &kvtest.Options{ 54 | NoLocks: opts.NoLocks, 55 | NoTx: true, // FIXME 56 | }) 57 | }) 58 | } 59 | 60 | var testList = []struct { 61 | name string 62 | test func(t *testing.T, db tuple.Store) 63 | }{ 64 | {name: "basic", test: basic}, 65 | {name: "typed", test: typed}, 66 | {name: "scans", test: scans}, 67 | {name: "tables", test: tables}, 68 | {name: "auto", test: auto}, 69 | } 70 | 71 | func basic(t *testing.T, db tuple.Store) { 72 | ctx := context.Background() 73 | tx, err := db.Tx(ctx, true) 74 | require.NoError(t, err) 75 | defer tx.Close() 76 | 77 | tbl, err := tx.CreateTable(ctx, tuple.Header{ 78 | Name: "test", 79 | Key: []tuple.KeyField{ 80 | {Name: "k1", Type: values.StringType{}}, 81 | }, 82 | Data: []tuple.Field{ 83 | {Name: "f1", Type: values.StringType{}}, 84 | }, 85 | }) 86 | require.NoError(t, err) 87 | 88 | k1 := tuple.SKey("a") 89 | v1 := tuple.SData("1") 90 | _, err = tbl.InsertTuple(ctx, tuple.Tuple{ 91 | Key: k1, Data: v1, 92 | }) 93 | require.NoError(t, err) 94 | 95 | v2, err := tbl.GetTuple(ctx, k1) 96 | require.NoError(t, err) 97 | require.Equal(t, v1, v2) 98 | 99 | it := tbl.Scan(ctx, &tuple.ScanOptions{Sort: tuple.SortAsc}) 100 | defer it.Close() 101 | 102 | var tuples []tuple.Tuple 103 | for it.Next(ctx) { 104 | tuples = append(tuples, tuple.Tuple{ 105 | Key: it.Key(), Data: it.Data(), 106 | }) 107 | } 108 | require.NoError(t, it.Err()) 109 | require.Equal(t, []tuple.Tuple{ 110 | {Key: k1, Data: v1}, 111 | }, tuples) 112 | } 113 | 114 | func typed(t *testing.T, db tuple.Store) { 115 | ctx := context.Background() 116 | tx, err := db.Tx(ctx, true) 117 | require.NoError(t, err) 118 | defer tx.Close() 119 | 120 | sortable := []values.Sortable{ 121 | values.String("foo"), 122 | values.Bytes("b\x00r"), 123 | values.Int(-42), 124 | values.UInt(42), 125 | values.Bool(false), 126 | // FIXME: test nanoseconds on backends that support it 127 | values.AsTime(time.Unix(123, 456789000)), 128 | } 129 | var payloads []values.Value 130 | for _, tp := range sortable { 131 | payloads = append(payloads, tp) 132 | } 133 | 134 | var ( 135 | kfields []tuple.KeyField 136 | kfields1 []tuple.KeyField 137 | vfields []tuple.Field 138 | 139 | key tuple.Key 140 | key1 tuple.Key 141 | data tuple.Data 142 | ) 143 | 144 | for i, v := range sortable { 145 | kf := tuple.KeyField{ 146 | Name: fmt.Sprintf("k%d", i+1), 147 | Type: v.SortableType(), 148 | } 149 | if i == 0 { 150 | key1 = append(key1, v) 151 | kfields1 = append(kfields1, kf) 152 | } 153 | key = append(key, v) 154 | kfields = append(kfields, kf) 155 | } 156 | for i, v := range payloads { 157 | data = append(data, v) 158 | vfields = append(vfields, tuple.Field{ 159 | Name: fmt.Sprintf("p%d", i+1), 160 | Type: v.Type(), 161 | }) 162 | } 163 | 164 | tbl1, err := tx.CreateTable(ctx, tuple.Header{ 165 | Name: "test1", Key: kfields1, Data: vfields, 166 | }) 167 | require.NoError(t, err, "\nkey : %#v\ndata: %#v", kfields1, vfields) 168 | tbl, err := tx.CreateTable(ctx, tuple.Header{ 169 | Name: "test2", Key: kfields, Data: vfields, 170 | }) 171 | require.NoError(t, err, "\nkey : %#v\ndata: %#v", kfields, vfields) 172 | 173 | _, err = tbl1.InsertTuple(ctx, tuple.Tuple{ 174 | Key: key1, Data: data, 175 | }) 176 | require.NoError(t, err) 177 | 178 | v2, err := tbl1.GetTuple(ctx, key1) 179 | require.NoError(t, err, "\nkey : %#v\ndata: %#v", kfields1, vfields) 180 | require.Equal(t, data, v2, "\n%v\nvs\n%v", data, v2) 181 | 182 | _, err = tbl.InsertTuple(ctx, tuple.Tuple{ 183 | Key: key, Data: data, 184 | }) 185 | require.NoError(t, err) 186 | 187 | v2, err = tbl.GetTuple(ctx, key) 188 | require.NoError(t, err, "\nkey : %#v\ndata: %#v", kfields, vfields) 189 | require.Equal(t, data, v2, "\n%v\nvs\n%v", data, v2) 190 | 191 | it := tbl.Scan(ctx, &tuple.ScanOptions{Sort: tuple.SortAsc}) 192 | defer it.Close() 193 | 194 | var tuples []tuple.Tuple 195 | for it.Next(ctx) { 196 | tuples = append(tuples, tuple.Tuple{ 197 | Key: it.Key(), Data: it.Data(), 198 | }) 199 | } 200 | require.NoError(t, it.Err()) 201 | require.Equal(t, []tuple.Tuple{ 202 | {Key: key, Data: data}, 203 | }, tuples) 204 | } 205 | 206 | func scans(t *testing.T, db tuple.Store) { 207 | ctx := context.Background() 208 | tx, err := db.Tx(ctx, true) 209 | require.NoError(t, err) 210 | defer tx.Close() 211 | 212 | tbl, err := tx.CreateTable(ctx, tuple.Header{ 213 | Name: "test", 214 | Key: []tuple.KeyField{ 215 | {Name: "k1", Type: values.StringType{}}, 216 | {Name: "k2", Type: values.StringType{}}, 217 | {Name: "k3", Type: values.StringType{}}, 218 | }, 219 | Data: []tuple.Field{ 220 | {Name: "f1", Type: values.IntType{}}, 221 | }, 222 | }) 223 | require.NoError(t, err) 224 | 225 | insert := func(key []string, n int) { 226 | var tkey tuple.Key 227 | for _, k := range key { 228 | tkey = append(tkey, values.String(k)) 229 | } 230 | _, err = tbl.InsertTuple(ctx, tuple.Tuple{ 231 | Key: tkey, Data: tuple.Data{values.Int(n)}, 232 | }) 233 | require.NoError(t, err) 234 | } 235 | 236 | scan := func(pref []string, exp ...int) { 237 | var kpref tuple.KeyFilters 238 | if len(pref) != 0 { 239 | for i, k := range pref { 240 | var f filter.SortableFilter 241 | if i == len(pref)-1 { 242 | if k == "" { 243 | break 244 | } 245 | f = filter.Prefix(values.String(k)) 246 | } else { 247 | f = filter.EQ(values.String(k)) 248 | } 249 | kpref = append(kpref, f) 250 | } 251 | } 252 | var f *tuple.Filter 253 | if kpref != nil { 254 | f = &tuple.Filter{KeyFilter: kpref} 255 | } 256 | it := tbl.Scan(ctx, &tuple.ScanOptions{Sort: tuple.SortAsc, Filter: f}) 257 | defer it.Close() 258 | 259 | var got []int 260 | for it.Next(ctx) { 261 | d := it.Data() 262 | require.True(t, len(d) == 1) 263 | v, ok := d[0].(values.Int) 264 | require.True(t, ok, "%T: %#v", d[0], d[0]) 265 | got = append(got, int(v)) 266 | } 267 | require.Equal(t, exp, got) 268 | } 269 | 270 | insert([]string{"a", "a", "a"}, 1) 271 | insert([]string{"b", "b", "b"}, 2) 272 | insert([]string{"a", "aa", "b"}, 3) 273 | insert([]string{"a", "ba", "c"}, 4) 274 | insert([]string{"a", "a", "ab"}, 5) 275 | insert([]string{"a", "b", "c"}, 6) 276 | 277 | scan(nil, 1, 5, 3, 6, 4, 2) 278 | scan([]string{""}, 1, 5, 3, 6, 4, 2) 279 | scan([]string{"a"}, 1, 5, 3, 6, 4) 280 | scan([]string{"b"}, 2) 281 | scan([]string{"a", "a"}, 1, 5, 3) 282 | scan([]string{"a", "a", ""}, 1, 5) 283 | scan([]string{"a", "aa"}, 3) 284 | scan([]string{"a", "aa", ""}, 3) 285 | scan([]string{"a", "aa", "b"}, 3) 286 | 287 | tbl2, err := tx.CreateTable(ctx, tuple.Header{ 288 | Name: "test2", 289 | Key: []tuple.KeyField{ 290 | {Name: "k1", Type: values.StringType{}}, 291 | {Name: "k2", Type: values.StringType{}}, 292 | {Name: "k3", Type: values.StringType{}}, 293 | }, 294 | Data: []tuple.Field{ 295 | {Name: "f1", Type: values.IntType{}}, 296 | }, 297 | }) 298 | require.NoError(t, err) 299 | 300 | _, err = tbl2.InsertTuple(ctx, tuple.Tuple{ 301 | Key: tuple.SKey("a", "b", "a"), Data: tuple.Data{values.Int(-1)}, 302 | }) 303 | require.NoError(t, err) 304 | 305 | scan(nil, 1, 5, 3, 6, 4, 2) 306 | } 307 | 308 | func tables(t *testing.T, db tuple.Store) { 309 | t.Run("simple", func(t *testing.T) { 310 | tablesSimple(t, db) 311 | }) 312 | t.Run("auto", func(t *testing.T) { 313 | tablesAuto(t, db) 314 | }) 315 | } 316 | 317 | func tablesSimple(t testing.TB, db tuple.Store) { 318 | ctx := context.Background() 319 | const name = "test1" 320 | 321 | schema := tuple.Header{ 322 | Name: name, 323 | Key: []tuple.KeyField{ 324 | {Name: "key1", Type: values.StringType{}}, 325 | }, 326 | Data: []tuple.Field{ 327 | {Name: "val1", Type: values.StringType{}}, 328 | }, 329 | } 330 | 331 | newTx := func(rw bool) tuple.Tx { 332 | tx, err := db.Tx(ctx, rw) 333 | require.NoError(t, err) 334 | return tx 335 | } 336 | 337 | tx := newTx(false) 338 | 339 | notExists := func() { 340 | tbl, err := tx.Table(ctx, name) 341 | require.Equal(t, tuple.ErrTableNotFound, err) 342 | require.Nil(t, tbl) 343 | } 344 | 345 | // access table when it not exists 346 | notExists() 347 | 348 | list, err := tx.ListTables(ctx) 349 | require.NoError(t, err) 350 | require.Empty(t, list) 351 | 352 | // create table on read-only transaction 353 | _, err = tx.CreateTable(ctx, schema) 354 | require.Equal(t, tuple.ErrReadOnly, err) 355 | 356 | notExists() 357 | 358 | err = tx.Close() 359 | require.NoError(t, err) 360 | 361 | // reopen read-write transaction 362 | tx = newTx(true) 363 | 364 | // table should not exist after failed creation 365 | tbl, err := tx.Table(ctx, name) 366 | require.Equal(t, tuple.ErrTableNotFound, err) 367 | require.Nil(t, tbl) 368 | 369 | tbl, err = tx.CreateTable(ctx, schema) 370 | require.NoError(t, err) 371 | require.NotNil(t, tbl) 372 | 373 | // TODO: check create + rollback 374 | err = tx.Commit(ctx) 375 | require.NoError(t, err) 376 | 377 | tx = newTx(true) 378 | 379 | tbl, err = tx.Table(ctx, name) 380 | require.NoError(t, err) 381 | assert.Equal(t, schema, tbl.Header()) 382 | 383 | err = tbl.Drop(ctx) 384 | require.NoError(t, err) 385 | 386 | tbl, err = tx.Table(ctx, name) 387 | require.Equal(t, tuple.ErrTableNotFound, err) 388 | require.Nil(t, tbl) 389 | 390 | err = tx.Commit(ctx) 391 | require.NoError(t, err) 392 | 393 | tx = newTx(false) 394 | 395 | notExists() 396 | 397 | err = tx.Close() 398 | require.NoError(t, err) 399 | 400 | // TODO: test multiple tables 401 | // TODO: test different headers (only keys, only values) 402 | } 403 | 404 | func tablesAuto(t testing.TB, db tuple.Store) { 405 | ctx := context.Background() 406 | const name = "test2" 407 | 408 | schema := tuple.Header{ 409 | Name: name, 410 | Key: []tuple.KeyField{ 411 | {Name: "key1", Type: values.UIntType{}, Auto: true}, 412 | }, 413 | Data: []tuple.Field{ 414 | {Name: "val1", Type: values.StringType{}}, 415 | }, 416 | } 417 | 418 | newTx := func(rw bool) tuple.Tx { 419 | tx, err := db.Tx(ctx, rw) 420 | require.NoError(t, err) 421 | return tx 422 | } 423 | 424 | tx := newTx(true) 425 | 426 | tbl, err := tx.CreateTable(ctx, schema) 427 | require.NoError(t, err) 428 | require.NotNil(t, tbl) 429 | 430 | err = tx.Commit(ctx) 431 | require.NoError(t, err) 432 | 433 | tx = newTx(false) 434 | 435 | tbl, err = tx.Table(ctx, name) 436 | require.NoError(t, err) 437 | assert.Equal(t, schema, tbl.Header()) 438 | 439 | err = tx.Close() 440 | require.NoError(t, err) 441 | } 442 | 443 | func auto(t *testing.T, db tuple.Store) { 444 | ctx := context.Background() 445 | tx, err := db.Tx(ctx, true) 446 | require.NoError(t, err) 447 | defer tx.Close() 448 | 449 | tbl, err := tx.CreateTable(ctx, tuple.Header{ 450 | Name: "test", 451 | Key: []tuple.KeyField{ 452 | {Name: "k1", Type: values.UIntType{}, Auto: true}, 453 | }, 454 | Data: []tuple.Field{ 455 | {Name: "f1", Type: values.StringType{}}, 456 | }, 457 | }) 458 | require.NoError(t, err) 459 | 460 | v1 := tuple.SData("1") 461 | k1, err := tbl.InsertTuple(ctx, tuple.Tuple{ 462 | Key: tuple.AutoKey(), Data: v1, 463 | }) 464 | require.NoError(t, err) 465 | require.Equal(t, 1, len(k1)) 466 | require.NotNil(t, k1[0]) 467 | 468 | v2, err := tbl.GetTuple(ctx, k1) 469 | require.NoError(t, err) 470 | require.Equal(t, v1, v2) 471 | 472 | it := tbl.Scan(ctx, &tuple.ScanOptions{Sort: tuple.SortAsc}) 473 | defer it.Close() 474 | 475 | var tuples []tuple.Tuple 476 | for it.Next(ctx) { 477 | tuples = append(tuples, tuple.Tuple{ 478 | Key: it.Key(), Data: it.Data(), 479 | }) 480 | } 481 | require.NoError(t, it.Err()) 482 | require.Equal(t, []tuple.Tuple{ 483 | {Key: k1, Data: v1}, 484 | }, tuples) 485 | } 486 | -------------------------------------------------------------------------------- /values/types.go: -------------------------------------------------------------------------------- 1 | package values 2 | 3 | type PrimitiveType interface { 4 | Type 5 | // NewPrimitive creates a new zero value of this type. 6 | NewPrimitive() PrimitiveDest 7 | } 8 | 9 | type Type interface { 10 | // New creates a new zero value of this type. 11 | New() ValueDest 12 | } 13 | 14 | type SortableType interface { 15 | Type 16 | // NewSortable creates a new zero value of this type. 17 | NewSortable() SortableDest 18 | } 19 | 20 | type BytesType struct{} 21 | 22 | func (tp BytesType) New() ValueDest { 23 | return tp.NewSortable() 24 | } 25 | 26 | func (BytesType) NewSortable() SortableDest { 27 | return new(Bytes) 28 | } 29 | 30 | func (BytesType) NewPrimitive() PrimitiveDest { 31 | return new(Bytes) 32 | } 33 | 34 | type StringType struct{} 35 | 36 | func (tp StringType) New() ValueDest { 37 | return tp.NewSortable() 38 | } 39 | 40 | func (StringType) NewSortable() SortableDest { 41 | return new(String) 42 | } 43 | 44 | func (StringType) NewPrimitive() PrimitiveDest { 45 | return new(String) 46 | } 47 | 48 | type IntType struct{} 49 | 50 | func (tp IntType) New() ValueDest { 51 | return tp.NewSortable() 52 | } 53 | 54 | func (IntType) NewSortable() SortableDest { 55 | return new(Int) 56 | } 57 | 58 | func (IntType) NewPrimitive() PrimitiveDest { 59 | return new(Int) 60 | } 61 | 62 | type UIntType struct{} 63 | 64 | func (tp UIntType) New() ValueDest { 65 | return tp.NewSortable() 66 | } 67 | 68 | func (UIntType) NewSortable() SortableDest { 69 | return new(UInt) 70 | } 71 | 72 | func (UIntType) NewPrimitive() PrimitiveDest { 73 | return new(UInt) 74 | } 75 | 76 | type BoolType struct{} 77 | 78 | func (tp BoolType) New() ValueDest { 79 | return tp.NewSortable() 80 | } 81 | 82 | func (BoolType) NewSortable() SortableDest { 83 | return new(Bool) 84 | } 85 | 86 | func (BoolType) NewPrimitive() PrimitiveDest { 87 | return new(Bool) 88 | } 89 | 90 | type TimeType struct{} 91 | 92 | func (tp TimeType) New() ValueDest { 93 | return tp.NewSortable() 94 | } 95 | 96 | func (TimeType) NewSortable() SortableDest { 97 | return new(Time) 98 | } 99 | 100 | type FloatType struct{} 101 | 102 | func (FloatType) New() ValueDest { 103 | return new(Float) 104 | } 105 | 106 | func (FloatType) NewPrimitive() PrimitiveDest { 107 | return new(Float) 108 | } 109 | -------------------------------------------------------------------------------- /values/values_test.go: -------------------------------------------------------------------------------- 1 | package values 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "reflect" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | var casesType = []struct { 14 | v Value 15 | n interface{} 16 | }{ 17 | {v: Bool(false), n: false}, 18 | {v: Int(0), n: int64(0)}, 19 | {v: UInt(0), n: uint64(0)}, 20 | {v: Float(0), n: float64(0)}, 21 | {v: String(""), n: ""}, 22 | {v: Bytes(nil), n: ([]byte)(nil)}, 23 | {v: Time{}, n: time.Time{}}, 24 | } 25 | 26 | func zeroOf(o interface{}) interface{} { 27 | return reflect.Zero(reflect.TypeOf(o)).Interface() 28 | } 29 | 30 | func TestTypes(t *testing.T) { 31 | for _, c := range casesType { 32 | t.Run(fmt.Sprintf("%T", c.v), func(t *testing.T) { 33 | require.Equal(t, c.n, c.v.Native()) 34 | v2 := c.v.Type().New().Value() 35 | require.Equal(t, c.v, v2) 36 | v2 = zeroOf(c.v.Type().New()).(ValueDest).Value() 37 | require.Equal(t, nil, v2) 38 | 39 | if p, ok := c.v.(Primitive); ok { 40 | p2 := p.PrimitiveType().NewPrimitive().Primitive() 41 | require.Equal(t, p, p2) 42 | p2 = zeroOf(p.PrimitiveType().NewPrimitive()).(PrimitiveDest).Primitive() 43 | require.Equal(t, nil, p2) 44 | } 45 | 46 | if s, ok := c.v.(Sortable); ok { 47 | s2 := s.SortableType().NewSortable().Sortable() 48 | require.Equal(t, s, s2) 49 | s2 = zeroOf(s.SortableType().NewSortable()).(SortableDest).Sortable() 50 | require.Equal(t, nil, s2) 51 | } 52 | }) 53 | } 54 | } 55 | 56 | var casesMarshalBinary = []struct { 57 | v Value 58 | }{ 59 | {v: Int(0)}, 60 | {v: Int(-1)}, 61 | {v: Int(+1)}, 62 | {v: Int(math.MinInt64)}, 63 | {v: Int(math.MaxInt64)}, 64 | {v: UInt(0)}, 65 | {v: UInt(+1)}, 66 | {v: UInt(math.MaxUint64)}, 67 | {v: Float(0)}, 68 | {v: Float(+1)}, 69 | {v: Float(-1)}, 70 | {v: Float(math.MaxFloat64)}, 71 | {v: Bool(false)}, 72 | {v: Bool(true)}, 73 | {v: String("")}, 74 | {v: String("abc")}, 75 | {v: Bytes{}}, 76 | {v: Bytes("\x00abc")}, 77 | {v: Time(time.Now().UTC())}, 78 | } 79 | 80 | func TestMarshalBinary(t *testing.T) { 81 | for _, c := range casesMarshalBinary { 82 | t.Run(fmt.Sprintf("%T(%v)", c.v, c.v), func(t *testing.T) { 83 | data, err := c.v.MarshalBinary() 84 | require.NoError(t, err) 85 | v2 := c.v.Type().New() 86 | err = v2.UnmarshalBinary(data) 87 | require.NoError(t, err) 88 | require.Equal(t, c.v, v2.Value()) 89 | 90 | if s, ok := c.v.(Sortable); ok { 91 | data, err = s.MarshalSortable() 92 | require.NoError(t, err) 93 | s2 := s.SortableType().NewSortable() 94 | err = s2.UnmarshalSortable(data) 95 | require.NoError(t, err) 96 | require.Equal(t, s, s2.Value()) 97 | } 98 | }) 99 | } 100 | } 101 | 102 | var casesCompare = []struct { 103 | name string 104 | a, b Sortable 105 | exp int 106 | }{ 107 | {name: "nil == nil", a: nil, b: nil, exp: 0}, 108 | {name: "nil < false", a: nil, b: Bool(false), exp: -1}, 109 | {name: "false < true", a: Bool(false), b: Bool(true), exp: -1}, 110 | {name: "nil < 1", a: nil, b: Int(1), exp: -1}, 111 | {name: "1 > nil", a: Int(1), b: nil, exp: +1}, 112 | {name: "1 == 1", a: Int(1), b: Int(1), exp: 0}, 113 | {name: "1 < 2", a: Int(1), b: Int(2), exp: -1}, 114 | {name: "-1 < 1", a: Int(-1), b: Int(+1), exp: -1}, 115 | {name: "-1 < 0", a: Int(-1), b: Int(0), exp: -1}, 116 | {name: "0 < 1", a: Int(0), b: Int(1), exp: -1}, 117 | {name: "min < min+1", a: Int(math.MinInt64), b: Int(math.MinInt64 + 1), exp: -1}, 118 | {name: "max-1 < max", a: Int(math.MaxInt64 - 1), b: Int(math.MaxInt64), exp: -1}, 119 | {name: "min < max", a: Int(math.MinInt64), b: Int(math.MaxInt64), exp: -1}, 120 | {name: "0 < umax", a: UInt(0), b: UInt(math.MaxUint64), exp: -1}, 121 | {name: "nil < a", a: nil, b: String("a"), exp: -1}, 122 | {name: "a > nil", a: String("a"), b: nil, exp: +1}, 123 | {name: "a == a", a: String("a"), b: String("a"), exp: 0}, 124 | {name: "a < b", a: String("a"), b: String("b"), exp: -1}, 125 | {name: "a < aa", a: String("a"), b: String("aa"), exp: -1}, 126 | {name: "nil < [a]", a: nil, b: Bytes("a"), exp: -1}, 127 | {name: "[] < [a]", a: Bytes{}, b: Bytes("a"), exp: -1}, 128 | {name: "[a] == [a]", a: Bytes("a"), b: Bytes("a"), exp: 0}, 129 | {name: "[a] < [b]", a: Bytes("a"), b: Bytes("b"), exp: -1}, 130 | {name: "[a] < [aa]", a: Bytes("a"), b: Bytes("aa"), exp: -1}, 131 | {name: "nil < t0", a: nil, b: Time{}, exp: -1}, 132 | {name: "t0-1 < t0", a: Time(time.Time{}.Add(-time.Hour)), b: Time{}, exp: -1}, 133 | {name: "now < now+1", a: AsTime(time.Now()), b: AsTime(time.Now().Add(time.Hour)), exp: -1}, 134 | } 135 | 136 | func TestCompare(t *testing.T) { 137 | for _, c := range casesCompare { 138 | t.Run(c.name, func(t *testing.T) { 139 | require.Equal(t, c.exp, Compare(c.a, c.b)) 140 | }) 141 | } 142 | } 143 | 144 | func TestIntSortable(t *testing.T) { 145 | cases := []struct { 146 | name string 147 | v int64 148 | exp uint64 149 | }{ 150 | {"min", math.MinInt64, 0}, 151 | {"min+1", math.MinInt64 + 1, 1}, 152 | {"0", 0, math.MaxInt64 + 1}, 153 | {"max-1", math.MaxInt64 - 1, math.MaxUint64 - 1}, 154 | {"max", math.MaxInt64, math.MaxUint64}, 155 | } 156 | for _, c := range cases { 157 | t.Run(c.name, func(t *testing.T) { 158 | uv := Int(c.v).asSortable() 159 | require.Equal(t, c.exp, uv, "conversion failed") 160 | var got Int 161 | got.setSortable(uv) 162 | require.Equal(t, c.v, int64(got), "reverse failed") 163 | }) 164 | } 165 | } 166 | --------------------------------------------------------------------------------