├── LICENSE ├── NOTICE.md ├── OSSMETADATA ├── README.md ├── example └── main.go ├── lmdbh └── lmdbh.go └── vendor └── github.com └── bmatsuo └── lmdb-go ├── README.md ├── cmd ├── lmdb_copy │ └── main.go └── lmdb_stat │ └── main.go ├── exp ├── cmd │ └── lmdb_cat │ │ └── main.go ├── lmdbscan │ ├── example_test.go │ ├── scanner.go │ └── scanner_test.go └── lmdbsync │ ├── handler.go │ ├── handler_test.go │ ├── lmdbsync.go │ ├── lmdbsync_test.go │ ├── testresize │ └── main.go │ └── testresize_test.go ├── internal ├── lmdbcmd │ └── cmutil.go └── lmdbtest │ └── lmdbtest.go └── lmdb ├── bench_test.go ├── cursor.go ├── cursor_test.go ├── env.go ├── env_test.go ├── error.go ├── error_test.go ├── error_unix.go ├── error_windows.go ├── example_test.go ├── lmdb.go ├── lmdb.h ├── lmdb_test.go ├── lmdbgo.c ├── lmdbgo.h ├── mdb.c ├── midl.c ├── midl.h ├── msgfunc.go ├── msgfunc_test.go ├── txn.go ├── txn_test.go ├── val.go └── val_test.go /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2016 Netflix, Inc. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /NOTICE.md: -------------------------------------------------------------------------------- 1 | # Licenses 2 | 3 | This project is built on the hard work of many people outside of Netflix, 4 | including Bryan Matsuo and the OpenLDAP authors and contributors. Below are the 5 | licenses for the projects on which rend-lmdb depends. 6 | 7 | ## Package lmdb-go by Bryan Matsuo 8 | 9 | Copyright (c) 2015, Bryan Matsuo 10 | All rights reserved. 11 | 12 | Redistribution and use in source and binary forms, with or without 13 | modification, are permitted provided that the following conditions are met: 14 | 15 | Redistributions of source code must retain the above copyright notice, this 16 | list of conditions and the following disclaimer. 17 | 18 | Redistributions in binary form must reproduce the above copyright notice, 19 | this list of conditions and the following disclaimer in the documentation 20 | and/or other materials provided with the distribution. 21 | 22 | Neither the name of the author nor the names of its contributors may be 23 | used to endorse or promote products derived from this software without specific 24 | prior written permission. 25 | 26 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 27 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 28 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 29 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 30 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 32 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 33 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 34 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 35 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 36 | 37 | 38 | ## LMDB itself, included as source in lmdb-go 39 | 40 | The OpenLDAP Public License 41 | Version 2.8, 17 August 2003 42 | 43 | Redistribution and use of this software and associated documentation 44 | ("Software"), with or without modification, are permitted provided 45 | that the following conditions are met: 46 | 47 | 1. Redistributions in source form must retain copyright statements 48 | and notices, 49 | 50 | 2. Redistributions in binary form must reproduce applicable copyright 51 | statements and notices, this list of conditions, and the following 52 | disclaimer in the documentation and/or other materials provided 53 | with the distribution, and 54 | 55 | 3. Redistributions must contain a verbatim copy of this document. 56 | 57 | The OpenLDAP Foundation may revise this license from time to time. 58 | Each revision is distinguished by a version number. You may use 59 | this Software under terms of this license revision or under the 60 | terms of any subsequent revision of the license. 61 | 62 | THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS 63 | CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, 64 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY 65 | AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 66 | SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S) 67 | OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, 68 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 69 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 70 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 71 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 72 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 73 | ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 74 | POSSIBILITY OF SUCH DAMAGE. 75 | 76 | The names of the authors and copyright holders must not be used in 77 | advertising or otherwise to promote the sale, use or other dealing 78 | in this Software without specific, written prior permission. Title 79 | to copyright in this Software shall at all times remain with copyright 80 | holders. 81 | 82 | OpenLDAP is a registered trademark of the OpenLDAP Foundation. 83 | 84 | Copyright 1999-2003 The OpenLDAP Foundation, Redwood City, 85 | California, USA. All Rights Reserved. Permission to copy and 86 | distribute verbatim copies of this document is granted. 87 | -------------------------------------------------------------------------------- /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=active 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rend LMDB backend 2 | 3 | This repository shows an example Rend server with an LMDB backend (as opposed to the standard one 4 | that speaks memcached). The code to create this version of Rend consists of an implementation of 5 | github.com/netflix/rend/handlers.Handler only. Beyond that, there is an example server 6 | implementation that shows how to wire up the LMDB backend with a Rend server. 7 | 8 | The Rend project can be used as a library as well as a server. The project has a default server 9 | which showcases the full capabilities it has, however using a library like this one, Rend can be 10 | extended to use any backend storage. 11 | 12 | The performance of Rend will be different based on the backend used, for example memcached stores 13 | data only in memory but this libary could possibly store data partially on disk and partially in 14 | memory - the same semantics as the underlying LMDB store. 15 | 16 | ## Prerequisites 17 | 18 | Latest version of Go, which can be found at https://golang.org/ 19 | 20 | ## Getting the code 21 | 22 | `$ go get github.com/netflix/rend-lmdb` 23 | 24 | ## Building the example server 25 | 26 | `$ go build github.com/netflix/rend-lmdb/example` 27 | 28 | ## Running the example server 29 | 30 | ``` 31 | $ umask 002 # For LMDB permissions 32 | $ ./example 33 | ``` 34 | 35 | ## Test it out 36 | 37 | Open another console window and try it out: 38 | 39 | ``` 40 | $ nc localhost 12121 41 | > get foo 42 | NOT_FOUND 43 | > set foo 0 0 6 44 | > foobar 45 | STORED 46 | > get foo 47 | END 48 | > touch foo 2 49 | TOUCHED 50 | > get foo 51 | VALUE foo 0 6 52 | foobar 53 | END 54 | > get foo 55 | END 56 | > quit 57 | Bye 58 | ``` 59 | -------------------------------------------------------------------------------- /example/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/netflix/rend-lmdb/lmdbh" 19 | "github.com/netflix/rend/handlers" 20 | "github.com/netflix/rend/orcas" 21 | "github.com/netflix/rend/server" 22 | ) 23 | 24 | func main() { 25 | largs := server.ListenArgs{ 26 | Type: server.ListenTCP, 27 | Port: 12121, 28 | } 29 | 30 | server.ListenAndServe( 31 | largs, 32 | server.Default, 33 | orcas.L1Only, 34 | lmdbh.New("/tmp/rendb/", 2*1024*1024*1024), 35 | handlers.NilHandler, 36 | ) 37 | } 38 | -------------------------------------------------------------------------------- /lmdbh/lmdbh.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Netflix, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package lmdbh 16 | 17 | import ( 18 | "encoding/binary" 19 | "log" 20 | "os" 21 | "sync" 22 | "time" 23 | 24 | "github.com/bmatsuo/lmdb-go/lmdb" 25 | "github.com/netflix/rend/common" 26 | "github.com/netflix/rend/handlers" 27 | ) 28 | 29 | type entry struct { 30 | exptime uint32 31 | flags uint32 32 | data []byte 33 | } 34 | 35 | func (e entry) expired() bool { 36 | return e.exptime != 0 && e.exptime < uint32(time.Now().Unix()) 37 | } 38 | 39 | func entryToBuf(e entry) []byte { 40 | // If this changes, make sure to update the GAT function below 41 | // The GAT function directly overwrites the exptime field 42 | buf := make([]byte, 8+len(e.data)) 43 | binary.BigEndian.PutUint32(buf[0:4], e.exptime) 44 | binary.BigEndian.PutUint32(buf[4:8], e.flags) 45 | copy(buf[8:], e.data) 46 | return buf 47 | } 48 | 49 | func bufToEntry(b []byte) entry { 50 | e := entry{ 51 | exptime: binary.BigEndian.Uint32(b[0:4]), 52 | flags: binary.BigEndian.Uint32(b[4:8]), 53 | data: make([]byte, len(b)-8), 54 | } 55 | 56 | copy(e.data, b[8:]) 57 | 58 | return e 59 | } 60 | 61 | func decode(err error) error { 62 | if err == nil { 63 | return err 64 | } 65 | 66 | if oe, ok := err.(*lmdb.OpError); ok { 67 | switch oe.Errno { 68 | case lmdb.KeyExist: //MDB_KEYEXIST 69 | return common.ErrKeyExists 70 | case lmdb.NotFound: //MDB_NOTFOUND 71 | return common.ErrKeyNotFound 72 | //case lmdb.PageNotFound: //MDB_PAGE_NOTFOUND 73 | //case lmdb.Corrupted: //MDB_CORRUPTED 74 | //case lmdb.Panic: //MDB_PANIC 75 | //case lmdb.VersionMismatch: //MDB_VERSION_MISMATCH 76 | //case lmdb.Invalid: //MDB_INVALID 77 | //case lmdb.MapFull: //MDB_MAP_FULL 78 | //case lmdb.DBsFull: //MDB_DBS_FULL 79 | //case lmdb.ReadersFull: //MDB_READERS_FULL 80 | //case lmdb.TLSFull: //MDB_TLS_FULL 81 | //case lmdb.TxnFull: //MDB_TXN_FULL 82 | //case lmdb.CursorFull: //MDB_CURSOR_FULL 83 | //case lmdb.PageFull: //MDB_PAGE_FULL 84 | //case lmdb.MapResized: //MDB_MAP_RESIZED 85 | //case lmdb.Incompatible: //MDB_INCOMPATIBLE 86 | //case lmdb.BadRSlot: //MDB_BAD_RSLOT 87 | //case lmdb.BadTxn: //MDB_BAD_TXN 88 | //case lmdb.BadValSize: //MDB_BAD_VALSIZE 89 | //case lmdb.BadDBI: //MDB_BAD_DBI 90 | // not sure is these should go here or if the return could be these 91 | //case syscall.EINVAL: 92 | //case syscall.EACCES: 93 | default: 94 | log.Println(err.Error) 95 | } 96 | } 97 | 98 | return err 99 | } 100 | 101 | type Handler struct { 102 | env *lmdb.Env 103 | dbi lmdb.DBI 104 | } 105 | 106 | var once = &sync.Once{} 107 | var singleton *Handler 108 | 109 | func reaper(env *lmdb.Env, dbi lmdb.DBI) { 110 | for { 111 | <-time.After(30 * time.Second) 112 | start := time.Now() 113 | log.Printf("[REAPER] Reaper started at %v\n", start) 114 | 115 | err := env.View(func(txn *lmdb.Txn) error { 116 | txn.RawRead = true 117 | cur, err := txn.OpenCursor(dbi) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | stats, err := txn.Stat(dbi) 123 | if err != nil { 124 | return err 125 | } 126 | log.Printf("[REAPER] Items before: %d", stats.Entries) 127 | 128 | // reap all items whose TTL has passed 129 | for { 130 | key, buf, err := cur.Get(nil, nil, lmdb.Next) 131 | if err != nil { 132 | if lmdb.IsNotFound(err) { 133 | break 134 | } 135 | return err 136 | } 137 | 138 | e := entry{ 139 | exptime: binary.BigEndian.Uint32(buf[0:4]), 140 | } 141 | if e.expired() { 142 | // Mini update transaction here to avoid blocking other writers 143 | err = env.Update(func(t *lmdb.Txn) error { 144 | // double check the expire time after getting txn lock 145 | buf, err := t.Get(dbi, key) 146 | if err != nil { 147 | return err 148 | } 149 | e := entry{ 150 | exptime: binary.BigEndian.Uint32(buf[0:4]), 151 | } 152 | if e.expired() { 153 | return t.Del(dbi, key, nil) 154 | } 155 | return nil 156 | }) 157 | 158 | if de := decode(err); de != nil && de != common.ErrKeyNotFound { 159 | return err 160 | } 161 | } 162 | } 163 | 164 | return nil 165 | }) 166 | 167 | if err != nil { 168 | log.Printf("[REAPER] Error while reaping: %v\n", err.Error()) 169 | } 170 | 171 | err = env.View(func(txn *lmdb.Txn) error { 172 | stats, err := txn.Stat(dbi) 173 | if err != nil { 174 | return err 175 | } 176 | log.Printf("[REAPER] Items after: %d\n", stats.Entries) 177 | return nil 178 | }) 179 | 180 | if err != nil { 181 | log.Printf("[REAPER] Error while reaping: %v\n", err.Error()) 182 | } 183 | 184 | end := time.Now() 185 | durms := float64(end.UnixNano()-start.UnixNano()) / 1000000.0 186 | log.Printf("[REAPER] Reaper ended at %v and took %vms to run\n", end, durms) 187 | } 188 | } 189 | 190 | func New(path string, size int64) handlers.HandlerConst { 191 | return func() (handlers.Handler, error) { 192 | once.Do(func() { 193 | // initialize the LMDB environment and DB 194 | env, err := lmdb.NewEnv() 195 | if err != nil { 196 | panic(err) 197 | } 198 | 199 | // apply size limit, one DB only 200 | if err := env.SetMapSize(size); err != nil { 201 | panic(err) 202 | } 203 | if err := env.SetMaxDBs(1); err != nil { 204 | panic(err) 205 | } 206 | 207 | // Create the db dir if it doesn't already exist 208 | fs, err := os.Stat(path) 209 | if err != nil { 210 | if os.IsNotExist(err) { 211 | if err := os.MkdirAll(path, 0774); err != nil { 212 | panic(err) 213 | } 214 | } else { 215 | panic(err) 216 | } 217 | } 218 | 219 | // Don't correct for a file already existing, let the user deal with it. 220 | if fs != nil && !fs.IsDir() { 221 | panic("Rend LMDB path exists and is a file") 222 | } 223 | 224 | if err := env.Open(path, 0, 0664); err != nil { 225 | panic(err) 226 | } 227 | 228 | var dbi lmdb.DBI 229 | err = env.Update(func(txn *lmdb.Txn) (err error) { 230 | dbi, err = txn.CreateDBI("rendb") 231 | return 232 | }) 233 | if err != nil { 234 | panic(err) 235 | } 236 | 237 | singleton = &Handler{ 238 | env: env, 239 | dbi: dbi, 240 | } 241 | 242 | go reaper(env, dbi) 243 | }) 244 | 245 | return singleton, nil 246 | } 247 | } 248 | 249 | func (h *Handler) Set(cmd common.SetRequest) error { 250 | var exptime uint32 251 | if cmd.Exptime > 0 { 252 | exptime = uint32(time.Now().Unix()) + cmd.Exptime 253 | } 254 | e := entry{ 255 | exptime: exptime, 256 | flags: cmd.Flags, 257 | data: cmd.Data, 258 | } 259 | 260 | buf := entryToBuf(e) 261 | 262 | err := h.env.Update(func(txn *lmdb.Txn) error { 263 | return txn.Put(h.dbi, cmd.Key, buf, 0) 264 | }) 265 | 266 | return decode(err) 267 | } 268 | 269 | func (h *Handler) Add(cmd common.SetRequest) error { 270 | var exptime uint32 271 | if cmd.Exptime > 0 { 272 | exptime = uint32(time.Now().Unix()) + cmd.Exptime 273 | } 274 | e := entry{ 275 | exptime: exptime, 276 | flags: cmd.Flags, 277 | data: cmd.Data, 278 | } 279 | 280 | buf := entryToBuf(e) 281 | 282 | err := h.env.Update(func(txn *lmdb.Txn) error { 283 | return txn.Put(h.dbi, cmd.Key, buf, lmdb.NoOverwrite) 284 | }) 285 | 286 | return decode(err) 287 | } 288 | 289 | func (h *Handler) Replace(cmd common.SetRequest) error { 290 | var exptime uint32 291 | if cmd.Exptime > 0 { 292 | exptime = uint32(time.Now().Unix()) + cmd.Exptime 293 | } 294 | e := entry{ 295 | exptime: exptime, 296 | flags: cmd.Flags, 297 | data: cmd.Data, 298 | } 299 | 300 | buf := entryToBuf(e) 301 | 302 | err := h.env.Update(func(txn *lmdb.Txn) error { 303 | if _, err := txn.Get(h.dbi, cmd.Key); err != nil { 304 | return err 305 | } 306 | 307 | return txn.Put(h.dbi, cmd.Key, buf, 0) 308 | }) 309 | 310 | return decode(err) 311 | } 312 | 313 | func (h *Handler) Append(cmd common.SetRequest) error { 314 | err := h.env.Update(func(txn *lmdb.Txn) error { 315 | buf, err := txn.Get(h.dbi, cmd.Key) 316 | if err != nil { 317 | return err 318 | } 319 | 320 | prev := bufToEntry(buf) 321 | 322 | e := entry{ 323 | exptime: prev.exptime, 324 | flags: prev.flags, 325 | data: append(prev.data, cmd.Data...), 326 | } 327 | 328 | buf = entryToBuf(e) 329 | 330 | return txn.Put(h.dbi, cmd.Key, buf, 0) 331 | }) 332 | 333 | return decode(err) 334 | } 335 | 336 | func (h *Handler) Prepend(cmd common.SetRequest) error { 337 | err := h.env.Update(func(txn *lmdb.Txn) error { 338 | buf, err := txn.Get(h.dbi, cmd.Key) 339 | if err != nil { 340 | return err 341 | } 342 | 343 | prev := bufToEntry(buf) 344 | 345 | e := entry{ 346 | exptime: prev.exptime, 347 | flags: prev.flags, 348 | data: append(cmd.Data, prev.data...), 349 | } 350 | 351 | buf = entryToBuf(e) 352 | 353 | return txn.Put(h.dbi, cmd.Key, buf, 0) 354 | }) 355 | 356 | return decode(err) 357 | } 358 | 359 | func (h *Handler) Get(cmd common.GetRequest) (<-chan common.GetResponse, <-chan error) { 360 | dataOut := make(chan common.GetResponse, len(cmd.Keys)) 361 | errorOut := make(chan error, 1) 362 | go realHandleGet(h, cmd, dataOut, errorOut) 363 | return dataOut, errorOut 364 | } 365 | 366 | func realHandleGet(h *Handler, cmd common.GetRequest, dataOut chan common.GetResponse, errorOut chan error) { 367 | err := h.env.View(func(txn *lmdb.Txn) error { 368 | for idx, key := range cmd.Keys { 369 | buf, err := txn.Get(h.dbi, key) 370 | if de := decode(err); de != nil { 371 | if de == common.ErrKeyNotFound { 372 | dataOut <- common.GetResponse{ 373 | Miss: true, 374 | Quiet: cmd.Quiet[idx], 375 | Opaque: cmd.Opaques[idx], 376 | Key: key, 377 | } 378 | continue 379 | } else { 380 | return de 381 | } 382 | } 383 | 384 | e := bufToEntry(buf) 385 | 386 | if e.expired() { 387 | dataOut <- common.GetResponse{ 388 | Miss: true, 389 | Quiet: cmd.Quiet[idx], 390 | Opaque: cmd.Opaques[idx], 391 | Key: key, 392 | } 393 | continue 394 | } 395 | 396 | dataOut <- common.GetResponse{ 397 | Miss: false, 398 | Quiet: cmd.Quiet[idx], 399 | Opaque: cmd.Opaques[idx], 400 | Flags: e.flags, 401 | Key: key, 402 | Data: e.data, 403 | } 404 | } 405 | return nil 406 | }) 407 | 408 | if err != nil { 409 | errorOut <- err 410 | } 411 | 412 | close(dataOut) 413 | close(errorOut) 414 | } 415 | 416 | func (h *Handler) GetE(cmd common.GetRequest) (<-chan common.GetEResponse, <-chan error) { 417 | dataOut := make(chan common.GetEResponse, len(cmd.Keys)) 418 | errorOut := make(chan error, 1) 419 | go realHandleGetE(h, cmd, dataOut, errorOut) 420 | return dataOut, errorOut 421 | } 422 | 423 | func realHandleGetE(h *Handler, cmd common.GetRequest, dataOut chan common.GetEResponse, errorOut chan error) { 424 | err := h.env.View(func(txn *lmdb.Txn) error { 425 | for idx, key := range cmd.Keys { 426 | buf, err := txn.Get(h.dbi, key) 427 | if de := decode(err); de != nil { 428 | if de == common.ErrKeyNotFound { 429 | dataOut <- common.GetEResponse{ 430 | Miss: true, 431 | Quiet: cmd.Quiet[idx], 432 | Opaque: cmd.Opaques[idx], 433 | Key: key, 434 | } 435 | continue 436 | } else { 437 | return de 438 | } 439 | } 440 | 441 | e := bufToEntry(buf) 442 | 443 | if e.expired() { 444 | dataOut <- common.GetEResponse{ 445 | Miss: true, 446 | Quiet: cmd.Quiet[idx], 447 | Opaque: cmd.Opaques[idx], 448 | Key: key, 449 | } 450 | continue 451 | } 452 | 453 | dataOut <- common.GetEResponse{ 454 | Miss: false, 455 | Quiet: cmd.Quiet[idx], 456 | Opaque: cmd.Opaques[idx], 457 | Exptime: e.exptime, 458 | Flags: e.flags, 459 | Key: key, 460 | Data: e.data, 461 | } 462 | } 463 | return nil 464 | }) 465 | 466 | if err != nil { 467 | errorOut <- err 468 | } 469 | 470 | close(dataOut) 471 | close(errorOut) 472 | } 473 | 474 | func (h *Handler) GAT(cmd common.GATRequest) (common.GetResponse, error) { 475 | var e entry 476 | 477 | err := h.env.Update(func(txn *lmdb.Txn) error { 478 | buf, err := txn.Get(h.dbi, cmd.Key) 479 | if err != nil { 480 | return err 481 | } 482 | 483 | e = bufToEntry(buf) 484 | 485 | // If the item is expired, proactively delete it 486 | if e.expired() { 487 | return txn.Del(h.dbi, cmd.Key, nil) 488 | } 489 | 490 | // set the new expiration time 491 | exptime := uint32(time.Now().Unix()) + cmd.Exptime 492 | binary.BigEndian.PutUint32(buf[0:4], exptime) 493 | 494 | return txn.Put(h.dbi, cmd.Key, buf, 0) 495 | }) 496 | 497 | if de := decode(err); de != nil { 498 | if de == common.ErrKeyNotFound { 499 | return common.GetResponse{ 500 | Miss: true, 501 | Opaque: cmd.Opaque, 502 | Key: cmd.Key, 503 | }, nil 504 | } else { 505 | return common.GetResponse{}, de 506 | } 507 | } 508 | 509 | return common.GetResponse{ 510 | Miss: false, 511 | Opaque: cmd.Opaque, 512 | Flags: e.flags, 513 | Key: cmd.Key, 514 | Data: e.data, 515 | }, nil 516 | } 517 | 518 | func (h *Handler) Delete(cmd common.DeleteRequest) error { 519 | err := h.env.Update(func(txn *lmdb.Txn) error { 520 | return txn.Del(h.dbi, cmd.Key, nil) 521 | }) 522 | 523 | return decode(err) 524 | } 525 | 526 | func (h *Handler) Touch(cmd common.TouchRequest) error { 527 | err := h.env.Update(func(txn *lmdb.Txn) error { 528 | buf, err := txn.Get(h.dbi, cmd.Key) 529 | if err != nil { 530 | return err 531 | } 532 | 533 | // set the new expiration time 534 | exptime := uint32(time.Now().Unix()) + cmd.Exptime 535 | binary.BigEndian.PutUint32(buf[0:4], exptime) 536 | 537 | return txn.Put(h.dbi, cmd.Key, buf, 0) 538 | }) 539 | 540 | return decode(err) 541 | } 542 | 543 | func (h *Handler) Close() error { 544 | // Singleton means don't close until the program shuts down 545 | return nil 546 | } 547 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/README.md: -------------------------------------------------------------------------------- 1 | #lmdb-go [![releases/v1.6.0](https://img.shields.io/badge/release-v1.6.0-375eab.svg)](CHANGES.md) [![C/v0.9.18](https://img.shields.io/badge/C-v0.9.18-555555.svg)](https://github.com/LMDB/lmdb/blob/mdb.RE/0.9/libraries/liblmdb/CHANGES) [![Build Status](https://travis-ci.org/bmatsuo/lmdb-go.svg?branch=master)](https://travis-ci.org/bmatsuo/lmdb-go) 2 | 3 | Go bindings to the OpenLDAP Lightning Memory-Mapped Database (LMDB). 4 | 5 | ## Packages 6 | 7 | Functionality is logically divided into several packages. Applications will 8 | usually need to import **lmdb** but may import other packages on an as needed 9 | basis. 10 | 11 | Packages in the `exp/` directory are not stable and may change without warning. 12 | That said, they are generally usable if application dependencies are managed 13 | and pinned by tag/commit. 14 | 15 | Developers concerned with package stability should consult the documentation. 16 | 17 | ####lmdb [![GoDoc](https://godoc.org/github.com/bmatsuo/lmdb-go/lmdb?status.svg)](https://godoc.org/github.com/bmatsuo/lmdb-go/lmdb) [![stable](https://img.shields.io/badge/stability-stable-brightgreen.svg)](#user-content-versioning-and-stability) [![GoCover](http://gocover.io/_badge/github.com/bmatsuo/lmdb-go/lmdb)](http://gocover.io/github.com/bmatsuo/lmdb-go/lmdb) 18 | 19 | ```go 20 | import "github.com/bmatsuo/lmdb-go/lmdb" 21 | ``` 22 | 23 | Core bindings allowing low-level access to LMDB. 24 | 25 | ####exp/lmdbscan [![GoDoc](https://godoc.org/github.com/bmatsuo/lmdb-go/exp/lmdbscan?status.svg)](https://godoc.org/github.com/bmatsuo/lmdb-go/exp/lmdbscan) [![unstable](https://img.shields.io/badge/stability-unstable-yellow.svg)](#user-content-versioning-and-stability) [![GoCover](http://gocover.io/_badge/github.com/bmatsuo/lmdb-go/exp/lmdbscan)](http://gocover.io/github.com/bmatsuo/lmdb-go/exp/lmdbscan) 26 | 27 | ```go 28 | import "github.com/bmatsuo/lmdb-go/exp/lmdbscan" 29 | ``` 30 | 31 | A utility package for scanning database ranges. The API is inspired by 32 | [bufio.Scanner](https://godoc.org/bufio#Scanner) and the python cursor 33 | [implementation](https://lmdb.readthedocs.org/en/release/#cursor-class). 34 | 35 | The **lmdbscan** package is unstable. The API is properly scoped and adequately 36 | tested. And no features that exist now will be removed without notice and a 37 | similar substitute. See the versioning documentation for more information. 38 | 39 | ####exp/lmdbsync [![GoDoc](https://godoc.org/github.com/bmatsuo/lmdb-go/exp/lmdbsync?status.svg)](https://godoc.org/github.com/bmatsuo/lmdb-go/exp/lmdbsync) [![experimental](https://img.shields.io/badge/stability-experimental-red.svg)](#user-content-versioning-and-stability) [![GoCover](http://gocover.io/_badge/github.com/bmatsuo/lmdb-go/exp/lmdbsync)](http://gocover.io/github.com/bmatsuo/lmdb-go/exp/lmdbsync) 40 | 41 | 42 | ```go 43 | import "github.com/bmatsuo/lmdb-go/exp/lmdbsync" 44 | ``` 45 | 46 | An experimental utility package that provides synchronization necessary to 47 | change an environment's map size after initialization. The package provides 48 | error handlers to automatically manage database size and retry failed 49 | transactions. 50 | 51 | The **lmdbsync** package is usable but the implementation of Handlers are 52 | unstable and may change in incompatible ways without notice. The use cases of 53 | dynamic map sizes and multiprocessing are niche and the package requires much 54 | more development driven by practical feedback before the Handler API and the 55 | provided implementations can be considered stable. 56 | 57 | ## Key Features 58 | 59 | ###Idiomatic API 60 | 61 | API inspired by [BoltDB](https://github.com/boltdb/bolt) with automatic 62 | commit/rollback of transactions. The goal of lmdb-go is to provide idiomatic, 63 | safe database interactions without compromising the flexibility of the C API. 64 | 65 | ###API coverage 66 | 67 | The lmdb-go project aims for complete coverage of the LMDB C API (within 68 | reason). Some notable features and optimizations that are supported: 69 | 70 | - Idiomatic subtransactions ("sub-updates") that do not disrupt thread locking. 71 | 72 | - Batch IO on databases utilizing the `MDB_DUPSORT` and `MDB_DUPFIXED` flags. 73 | 74 | - Reserved writes than can save in memory copies converting/buffering into 75 | `[]byte`. 76 | 77 | For tracking purposes a list of unsupported features is kept in an 78 | [issue](https://github.com/bmatsuo/lmdb-go/issues/1). 79 | 80 | ###Zero-copy reads 81 | 82 | Applications with high performance requirements can opt-in to fast, zero-copy 83 | reads at the cost of runtime safety. Zero-copy behavior is specified at the 84 | transaction level to reduce instrumentation overhead. 85 | 86 | ``` 87 | err := lmdb.View(func(txn *lmdb.Txn) error { 88 | // RawRead enables zero-copy behavior with some serious caveats. 89 | // Read the documentation carefully before using. 90 | txn.RawRead = true 91 | 92 | val, err := txn.Get(dbi, []byte("largevalue"), 0) 93 | // ... 94 | }) 95 | ``` 96 | 97 | ###Documentation 98 | 99 | Comprehensive documentation and examples are provided to demonstrate safe usage 100 | of lmdb. In addition to [godoc](https://godoc.org/github.com/bmatsuo/lmdb-go) 101 | documentation, implementations of the standand LMDB commands (`mdb_stat`, etc) 102 | can be found in the [cmd/](cmd/) directory and some simple experimental 103 | commands can be found in the [exp/cmd/](exp/cmd) directory. Aside from 104 | providing minor utility these programs are provided as examples of lmdb in 105 | practice. 106 | 107 | ##LMDB compared to BoltDB 108 | 109 | BoltDB is a quality database with a design similar to LMDB. Both store 110 | key-value data in a file and provide ACID transactions. So there are often 111 | questions of why to use one database or the other. 112 | 113 | ###Advantages of BoltDB 114 | 115 | - Nested databases allow for hierarchical data organization. 116 | 117 | - Far more databases can be accessed concurrently. 118 | 119 | - Operating systems that do not support sparse files do not use up excessive 120 | space due to a large pre-allocation of file space. The exp/lmdbsync package 121 | is intended to resolve this problem with LMDB but it is not ready. 122 | 123 | - As a pure Go package bolt can be easily cross-compiled using the `go` 124 | toolchain and `GOOS`/`GOARCH` variables. 125 | 126 | ###Advantages of LMDB 127 | 128 | - Keys can contain multiple values using the DupSort flag. 129 | 130 | - Updates can have sub-updates for atomic batching of changes. 131 | 132 | - Databases typically remain open for the application lifetime. This limits 133 | the number of concurrently accessible databases. But, this minimizes the 134 | overhead of database accesses and typically produces cleaner code than 135 | an equivalent BoltDB implementation. 136 | 137 | - Significantly faster than BoltDB. The raw speed of LMDB easily surpasses 138 | BoltDB. Additionally, LMDB provides optimizations ranging from safe, 139 | feature-specific optimizations to generally unsafe, extremely situational 140 | ones. Applications are free to enable any optimizations that fit their data, 141 | access, and reliability models. 142 | 143 | - LMDB allows multiple applications to access a database simultaneously. 144 | Updates from concurrent processes are synchronized using a database lock 145 | file. 146 | 147 | - As a C library, applications in any language can interact with LMDB 148 | databases. Mission critical Go applications can use a database while Python 149 | scripts perform analysis on the side. 150 | 151 | ##Build 152 | 153 | There is no dependency on shared libraries. So most users can simply install 154 | using `go get`. 155 | 156 | `go get github.com/bmatsuo/lmdb-go/lmdb` 157 | 158 | On FreeBSD 10, you must explicitly set `CC` (otherwise it will fail with a 159 | cryptic error), for example: 160 | 161 | CC=clang go test -v ./... 162 | 163 | Building commands and running tests can be done with `go` or with `make` 164 | 165 | make bin 166 | make test 167 | make check 168 | make all 169 | 170 | On Linux, you can specify the `pwritev` build tag to reduce the number of syscalls 171 | required when committing a transaction. In your own package you can then do 172 | 173 | go build -tags pwritev . 174 | 175 | to enable the optimisation. 176 | 177 | ##Documentation 178 | 179 | ###Go doc 180 | 181 | The `go doc` documentation available on 182 | [godoc.org](https://godoc.org/github.com/bmatsuo/lmdb-go) is the primary source 183 | of developer documentation for lmdb-go. It provides an overview of the API 184 | with a lot of usage examples. Where necessary the documentation points out 185 | differences between the semantics of methods and their C counterparts. 186 | 187 | ###LMDB 188 | 189 | The LMDB [homepage](http://symas.com/mdb/) and mailing list 190 | ([archives](http://www.openldap.org/lists/openldap-technical/)) are the 191 | official source of documentation regarding low-level LMDB operation and 192 | internals. 193 | 194 | Along with an API reference LMDB provides a high-level 195 | [summary](http://symas.com/mdb/doc/starting.html) of the library. While 196 | lmdb-go abstracts many of the thread and transaction details by default the 197 | rest of the guide is still useful to compare with `go doc`. 198 | 199 | ###Versioning and Stability 200 | 201 | The lmdb-go project makes regular releases with IDs `X.Y.Z`. All packages 202 | outside of the `exp/` directory are considered stable and adhere to the 203 | guidelines of [semantic versioning](http://semver.org/). 204 | 205 | Experimental packages (those packages in `exp/`) are not required to adhere to 206 | semantic versioning. However packages specifically declared to merely be 207 | "unstable" can be relied on more for long term use with less concern. 208 | 209 | The API of an unstable package may change in subtle ways between minor release 210 | versions. But deprecations will be indicated at least one release in advance 211 | and all functionality will remain available through some method. 212 | 213 | ##License 214 | 215 | Except where otherwise noted files in the lmdb-go project are licensed under 216 | the MIT open source license. 217 | 218 | The LMDB C source is licensed under the OpenLDAP Public License. 219 | 220 | ##Links 221 | 222 | ####[github.com/bmatsuo/raft-mdb](https://github.com/bmatsuo/raft-mdb) ([godoc](https://godoc.org/github.com/bmatsuo/raft-mdb)) 223 | 224 | An experimental backend for 225 | [github.com/hashicorp/raft](https://github.com/hashicorp/raft) forked from 226 | [github.com/hashicorp/raft-mdb](https://github.com/hashicorp/raft-mdb). 227 | 228 | ####[github.com/bmatsuo/cayley/graph/lmdb](https://github.com/bmatsuo/cayley/tree/master/graph/lmdb) ([godoc](https://godoc.org/github.com/bmatsuo/cayley/graph/lmdb)) 229 | 230 | Experimental backend quad-store for 231 | [github.com/google/cayley](https://github.com/google/cayley) based off of the 232 | BoltDB 233 | [implementation](https://github.com/google/cayley/tree/master/graph/bolt). 234 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/cmd/lmdb_copy/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Command lmdb_copy is a clone of mdb_copy that copies an LMDB environment. A 3 | consistent copy is made even if the source database is in use. 4 | 5 | Command line flags mirror the flags for the original program. For information 6 | about, run lmdb_copy with the -h flag. 7 | 8 | lmdb_copy -h 9 | */ 10 | package main 11 | 12 | import ( 13 | "flag" 14 | "log" 15 | "os" 16 | 17 | "github.com/bmatsuo/lmdb-go/internal/lmdbcmd" 18 | "github.com/bmatsuo/lmdb-go/lmdb" 19 | ) 20 | 21 | func main() { 22 | opt := &Options{} 23 | flag.BoolVar(&opt.Compact, "c", false, "Compact while copying.") 24 | flag.Parse() 25 | 26 | lmdbcmd.PrintVersion() 27 | 28 | if flag.NArg() > 2 { 29 | log.Fatalf("too many arguments specified") 30 | } 31 | if flag.NArg() == 0 { 32 | log.Fatalf("at least one argument must be specified") 33 | } 34 | 35 | var srcpath, dstpath string 36 | srcpath = flag.Arg(0) 37 | if flag.NArg() > 1 { 38 | dstpath = flag.Arg(1) 39 | } 40 | 41 | copyEnv(srcpath, dstpath, opt) 42 | } 43 | 44 | // Options contain the command line options for an lmdb_copy command. 45 | type Options struct { 46 | Compact bool 47 | } 48 | 49 | func copyEnv(srcpath, dstpath string, opt *Options) error { 50 | env, err := lmdb.NewEnv() 51 | if err != nil { 52 | return err 53 | } 54 | err = env.Open(srcpath, lmdbcmd.OpenFlag(), 0644) 55 | if err != nil { 56 | return err 57 | } 58 | var flags uint 59 | if opt != nil && opt.Compact { 60 | flags |= lmdb.CopyCompact 61 | } 62 | if dstpath != "" { 63 | return env.CopyFlag(dstpath, flags) 64 | } 65 | fd := os.Stdout.Fd() 66 | return env.CopyFDFlag(fd, flags) 67 | } 68 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/cmd/lmdb_stat/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Command lmdb_stat is a clone of mdb_stat that displays the status an LMDB 3 | environment. 4 | 5 | Command line flags mirror the flags for the original program. For information 6 | about, run lmdb_stat with the -h flag. 7 | 8 | lmdb_stat -h 9 | */ 10 | package main 11 | 12 | import "C" 13 | 14 | import ( 15 | "bufio" 16 | "flag" 17 | "fmt" 18 | "io" 19 | "log" 20 | "os" 21 | "reflect" 22 | "strings" 23 | "unsafe" 24 | 25 | "github.com/bmatsuo/lmdb-go/exp/lmdbscan" 26 | "github.com/bmatsuo/lmdb-go/internal/lmdbcmd" 27 | "github.com/bmatsuo/lmdb-go/lmdb" 28 | ) 29 | 30 | func main() { 31 | opt := &Options{} 32 | flag.BoolVar(&opt.PrintInfo, "e", false, "Display information about the database environment") 33 | flag.BoolVar(&opt.PrintFree, "f", false, "Display freelist information") 34 | flag.BoolVar(&opt.PrintFreeSummary, "ff", false, "Display freelist information") 35 | flag.BoolVar(&opt.PrintFreeFull, "fff", false, "Display freelist information") 36 | flag.BoolVar(&opt.PrintReaders, "r", false, strings.Join([]string{ 37 | "Display information about the environment reader table.", 38 | "Shows the process ID, thread ID, and transaction ID for each active reader slot.", 39 | }, " ")) 40 | flag.BoolVar(&opt.PrintReadersCheck, "rr", false, strings.Join([]string{ 41 | "Implies -r.", 42 | "Check for stale entries in the reader table and clear them.", 43 | "The reader table is printed again after the check is performed.", 44 | }, " ")) 45 | flag.BoolVar(&opt.PrintStatAll, "a", false, "Display the status of all databases in the environment") 46 | flag.StringVar(&opt.PrintStatSub, "s", "", "Display the status of a specific subdatabase.") 47 | flag.BoolVar(&opt.Debug, "D", false, "print debug information") 48 | flag.Parse() 49 | 50 | lmdbcmd.PrintVersion() 51 | 52 | if opt.PrintStatAll && opt.PrintStatSub != "" { 53 | log.Fatal("only one of -a and -s may be provided") 54 | } 55 | 56 | if flag.NArg() > 1 { 57 | log.Fatalf("too many argument provided") 58 | } 59 | if flag.NArg() == 0 { 60 | log.Fatalf("missing argument") 61 | } 62 | opt.Path = flag.Arg(0) 63 | 64 | var failed bool 65 | defer func() { 66 | if e := recover(); e != nil { 67 | if opt.Debug { 68 | panic(e) 69 | } 70 | log.Print(e) 71 | failed = true 72 | } 73 | if failed { 74 | os.Exit(1) 75 | } 76 | }() 77 | 78 | err := doMain(opt) 79 | if err != nil { 80 | log.Print(err) 81 | failed = true 82 | } 83 | } 84 | 85 | // Options contains all the configuration for an lmdb_stat command including 86 | // command line arguments. 87 | type Options struct { 88 | Path string 89 | 90 | PrintInfo bool 91 | PrintReaders bool 92 | PrintReadersCheck bool 93 | PrintFree bool 94 | PrintFreeSummary bool 95 | PrintFreeFull bool 96 | PrintStatAll bool 97 | PrintStatSub string 98 | 99 | Debug bool 100 | } 101 | 102 | func doMain(opt *Options) error { 103 | env, err := lmdb.NewEnv() 104 | if err != nil { 105 | return err 106 | } 107 | if opt.PrintStatAll || opt.PrintStatSub != "" { 108 | err = env.SetMaxDBs(1) 109 | if err != nil { 110 | return err 111 | } 112 | } 113 | err = env.Open(opt.Path, lmdbcmd.OpenFlag(), 0644) 114 | defer env.Close() 115 | if err != nil { 116 | return err 117 | } 118 | 119 | if opt.PrintInfo { 120 | err = doPrintInfo(env, opt) 121 | if err != nil { 122 | return err 123 | } 124 | } 125 | 126 | if opt.PrintReaders || opt.PrintReadersCheck { 127 | err = doPrintReaders(env, opt) 128 | if err != nil { 129 | return err 130 | } 131 | } 132 | 133 | if opt.PrintFree || opt.PrintFreeSummary || opt.PrintFreeFull { 134 | err = doPrintFree(env, opt) 135 | if err != nil { 136 | return err 137 | } 138 | } 139 | 140 | err = doPrintStatRoot(env, opt) 141 | if err != nil { 142 | return err 143 | } 144 | 145 | if opt.PrintStatAll { 146 | err = doPrintStatAll(env, opt) 147 | if err != nil { 148 | return err 149 | } 150 | } else if opt.PrintStatSub != "" { 151 | err = doPrintStatDB(env, opt.PrintStatSub, opt) 152 | if err != nil { 153 | return err 154 | } 155 | } 156 | 157 | return nil 158 | } 159 | 160 | func doPrintInfo(env *lmdb.Env, opt *Options) error { 161 | info, err := env.Info() 162 | if err != nil { 163 | return err 164 | } 165 | 166 | pagesize := os.Getpagesize() 167 | 168 | fmt.Println("Environment Info") 169 | fmt.Println(" Map address:", nil) 170 | fmt.Println(" Map size:", info.MapSize) 171 | fmt.Println(" Page size:", pagesize) 172 | fmt.Println(" Max pages:", info.MapSize/int64(pagesize)) 173 | fmt.Println(" Number of pages used:", info.LastPNO+1) 174 | fmt.Println(" Last transaction ID:", info.LastTxnID) 175 | fmt.Println(" Max readers:", info.MaxReaders) 176 | fmt.Println(" Number of readers used:", info.NumReaders) 177 | 178 | return nil 179 | } 180 | 181 | func doPrintReaders(env *lmdb.Env, opt *Options) error { 182 | fmt.Println("Reader Table Status") 183 | w := bufio.NewWriter(os.Stdout) 184 | err := printReaders(env, w, opt) 185 | if err == nil { 186 | err = w.Flush() 187 | } 188 | if err != nil { 189 | return err 190 | } 191 | 192 | if opt.PrintReadersCheck { 193 | numstale, err := env.ReaderCheck() 194 | if err != nil { 195 | return err 196 | } 197 | fmt.Printf(" %d stale readers cleared.\n", numstale) 198 | err = printReaders(env, w, opt) 199 | if err == nil { 200 | err = w.Flush() 201 | } 202 | if err != nil { 203 | return err 204 | } 205 | } 206 | 207 | return nil 208 | } 209 | 210 | func printReaders(env *lmdb.Env, w io.Writer, opt *Options) error { 211 | return env.ReaderList(func(msg string) error { 212 | _, err := fmt.Fprint(w, msg) 213 | return err 214 | }) 215 | } 216 | 217 | func doPrintFree(env *lmdb.Env, opt *Options) error { 218 | return env.View(func(txn *lmdb.Txn) (err error) { 219 | txn.RawRead = true 220 | 221 | fmt.Println("Freelist Status") 222 | 223 | stat, err := txn.Stat(0) 224 | if err != nil { 225 | return err 226 | } 227 | printStat(stat, opt) 228 | 229 | var numpages int64 230 | s := lmdbscan.New(txn, 0) 231 | defer s.Close() 232 | for s.Scan() { 233 | key := s.Key() 234 | data := s.Val() 235 | txid := *(*C.size_t)(unsafe.Pointer(&key[0])) 236 | ipages := int64(*(*C.size_t)(unsafe.Pointer(&data[0]))) 237 | numpages += ipages 238 | if opt.PrintFreeSummary || opt.PrintFreeFull { 239 | bad := "" 240 | hdr := reflect.SliceHeader{ 241 | Data: uintptr(unsafe.Pointer(&data[0])), 242 | Len: int(ipages) + 1, 243 | Cap: int(ipages) + 1, 244 | } 245 | pages := *(*[]C.size_t)(unsafe.Pointer(&hdr)) 246 | pages = pages[1:] 247 | var span C.size_t 248 | prev := C.size_t(1) 249 | for i := ipages - 1; i >= 0; i-- { 250 | pg := pages[i] 251 | if pg < prev { 252 | bad = " [bad sequence]" 253 | } 254 | prev = pg 255 | pg += span 256 | for i >= int64(span) && pages[i-int64(span)] == pg { 257 | span++ 258 | pg++ 259 | } 260 | } 261 | fmt.Printf(" Transaction %d, %d pages, maxspan %d%s\n", txid, ipages, span, bad) 262 | 263 | if opt.PrintFreeFull { 264 | for j := ipages - 1; j >= 0; { 265 | pg := pages[j] 266 | j-- 267 | span := C.size_t(1) 268 | for j >= 0 && pages[j] == pg+span { 269 | j-- 270 | span++ 271 | } 272 | if span > 1 { 273 | fmt.Printf(" %9d[%d]\n", pg, span) 274 | } else { 275 | fmt.Printf(" %9d\n", pg) 276 | } 277 | } 278 | } 279 | } 280 | } 281 | err = s.Err() 282 | if err != nil { 283 | return err 284 | } 285 | 286 | fmt.Println(" Free pages:", numpages) 287 | 288 | return nil 289 | }) 290 | } 291 | 292 | func doPrintStatRoot(env *lmdb.Env, opt *Options) error { 293 | stat, err := env.Stat() 294 | if err != nil { 295 | return err 296 | } 297 | 298 | fmt.Println("Status of Main DB") 299 | fmt.Println(" Tree depth:", stat.Depth) 300 | fmt.Println(" Branch pages:", stat.BranchPages) 301 | fmt.Println(" Leaf pages:", stat.LeafPages) 302 | fmt.Println(" Overflow pages:", stat.OverflowPages) 303 | fmt.Println(" Entries:", stat.Entries) 304 | 305 | return nil 306 | } 307 | 308 | func doPrintStatDB(env *lmdb.Env, db string, opt *Options) error { 309 | err := env.View(func(txn *lmdb.Txn) (err error) { 310 | return printStatDB(env, txn, db, opt) 311 | }) 312 | if err != nil { 313 | return fmt.Errorf("%v (%s)", err, db) 314 | } 315 | return nil 316 | } 317 | 318 | func printStatDB(env *lmdb.Env, txn *lmdb.Txn, db string, opt *Options) error { 319 | dbi, err := txn.OpenDBI(db, 0) 320 | if err != nil { 321 | return err 322 | } 323 | defer env.CloseDBI(dbi) 324 | 325 | stat, err := txn.Stat(dbi) 326 | if err != nil { 327 | return err 328 | } 329 | 330 | fmt.Println("Status of", db) 331 | printStat(stat, opt) 332 | 333 | return err 334 | } 335 | 336 | func printStat(stat *lmdb.Stat, opt *Options) error { 337 | fmt.Println(" Tree depth:", stat.Depth) 338 | fmt.Println(" Branch pages:", stat.BranchPages) 339 | fmt.Println(" Leaf pages:", stat.LeafPages) 340 | fmt.Println(" Overflow pages:", stat.OverflowPages) 341 | fmt.Println(" Entries:", stat.Entries) 342 | 343 | return nil 344 | } 345 | 346 | func doPrintStatAll(env *lmdb.Env, opt *Options) error { 347 | return env.View(func(txn *lmdb.Txn) (err error) { 348 | dbi, err := txn.OpenRoot(0) 349 | if err != nil { 350 | return err 351 | } 352 | defer env.CloseDBI(dbi) 353 | 354 | s := lmdbscan.New(txn, dbi) 355 | defer s.Close() 356 | for s.Scan() { 357 | err = printStatDB(env, txn, string(s.Key()), opt) 358 | if e, ok := err.(*lmdb.OpError); ok { 359 | if e.Op == "mdb_dbi_open" { 360 | continue 361 | } 362 | } 363 | if err != nil { 364 | return fmt.Errorf("%v (%s)", err, s.Key()) 365 | } 366 | } 367 | return s.Err() 368 | }) 369 | } 370 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/exp/cmd/lmdb_cat/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "log" 10 | "os" 11 | "strings" 12 | 13 | "github.com/bmatsuo/lmdb-go/exp/lmdbscan" 14 | "github.com/bmatsuo/lmdb-go/exp/lmdbsync" 15 | "github.com/bmatsuo/lmdb-go/internal/lmdbcmd" 16 | "github.com/bmatsuo/lmdb-go/lmdb" 17 | ) 18 | 19 | func main() { 20 | opt := &Options{} 21 | flag.BoolVar(&opt.ReadIn, "i", false, "Read data from standard input and write it the specefied database.") 22 | flag.BoolVar(&opt.KeyOnly, "k", false, "Do not write value data to standard output") 23 | flag.BoolVar(&opt.ValOnly, "K", false, "Do not write key data to standard output") 24 | flag.StringVar(&opt.Sep, "F", "=", "Key-value delimiter for items written to standard output, or read from standard output.") 25 | flag.Parse() 26 | 27 | lmdbcmd.PrintVersion() 28 | 29 | dbs := flag.Args() 30 | var specs []*catSpec 31 | for _, db := range dbs { 32 | spec, err := parseCatSpec(db) 33 | if err != nil { 34 | log.Fatal(err) 35 | } 36 | specs = append(specs, spec) 37 | } 38 | 39 | if opt.ReadIn { 40 | if len(specs) > 1 || len(specs[0].DB) > 1 { 41 | log.Fatalf("only one database may be specefied when -i is given") 42 | } 43 | if len(specs[0].DB) > 0 { 44 | opt.DB = specs[0].DB[0] 45 | } 46 | if opt.KeyOnly || opt.ValOnly { 47 | log.Fatal("flags -k and -K must be omitted when -i is given") 48 | } 49 | if opt.Sep == "" { 50 | log.Fatal("delimiter -F cannot be empty when -i is given") 51 | } 52 | 53 | err := readIn(specs[0].Path, os.Stdin, opt) 54 | if err != nil { 55 | log.Fatal(err) 56 | } 57 | return 58 | } 59 | 60 | for _, spec := range specs { 61 | opt := &catOptions{DB: spec.DB} 62 | cat(spec.Path, opt) 63 | } 64 | } 65 | 66 | // Options contains the the configuration to perform an lmdb_cat on a single 67 | // environment/database. 68 | type Options struct { 69 | ReadIn bool 70 | KeyOnly bool 71 | ValOnly bool 72 | Sep string 73 | DB string 74 | } 75 | 76 | func readIn(path string, r io.Reader, opt *Options) error { 77 | _env, err := lmdb.NewEnv() 78 | if err != nil { 79 | return err 80 | } 81 | err = _env.SetMapSize(100 << 10) 82 | if err != nil { 83 | return err 84 | } 85 | if opt != nil && opt.DB != "" { 86 | err = _env.SetMaxDBs(1) 87 | if err != nil { 88 | return err 89 | } 90 | } 91 | err = _env.Open(path, lmdbcmd.OpenFlag(), 0644) 92 | defer _env.Close() 93 | if err != nil { 94 | return err 95 | } 96 | doubleSize := func(size int64) (int64, bool) { return size * 2, true } 97 | handler := lmdbsync.MapFullHandler(doubleSize) 98 | env, err := lmdbsync.NewEnv(_env, handler) 99 | if err != nil { 100 | return err 101 | } 102 | return env.Update(func(txn *lmdb.Txn) (err error) { 103 | var dbi lmdb.DBI 104 | if opt.DB == "" { 105 | dbi, err = txn.OpenRoot(0) 106 | } else { 107 | dbi, err = txn.OpenDBI(opt.DB, lmdb.Create) 108 | } 109 | if err != nil { 110 | return err 111 | } 112 | 113 | cur, err := txn.OpenCursor(dbi) 114 | if err != nil { 115 | return err 116 | } 117 | defer cur.Close() 118 | 119 | sep := []byte(opt.Sep) 120 | 121 | numln := 0 122 | s := bufio.NewScanner(r) 123 | for s.Scan() { 124 | numln++ 125 | ln := s.Bytes() 126 | pieces := bytes.SplitN(ln, sep, 2) 127 | if len(pieces) < 2 { 128 | log.Printf("line %d: missing separator", numln) 129 | continue 130 | } 131 | err = cur.Put(pieces[0], pieces[1], 0) 132 | if err != nil { 133 | return err 134 | } 135 | } 136 | return s.Err() 137 | }) 138 | } 139 | 140 | type catSpec struct { 141 | Path string 142 | DB []string 143 | } 144 | 145 | // BUG: 146 | // this function is shit 147 | func parseCatSpec(s string) (*catSpec, error) { 148 | s = strings.TrimSpace(s) 149 | dbspec := strings.Index(s, "[") 150 | if dbspec < 0 { 151 | spec := &catSpec{Path: s} 152 | return spec, nil 153 | } 154 | if !strings.HasSuffix(s, "]") { 155 | return nil, fmt.Errorf("invalid db spec") 156 | } 157 | dbs := strings.Split(s[dbspec+1:len(s)-1], ":") 158 | spec := &catSpec{Path: s[:dbspec], DB: dbs} 159 | return spec, nil 160 | } 161 | 162 | type catOptions struct { 163 | DB []string 164 | } 165 | 166 | func cat(path string, opt *catOptions) error { 167 | env, err := lmdb.NewEnv() 168 | if err != nil { 169 | return err 170 | } 171 | maxdbs := 0 172 | if opt != nil && len(opt.DB) > 0 { 173 | maxdbs = len(opt.DB) 174 | } 175 | if maxdbs > 0 { 176 | err = env.SetMaxDBs(maxdbs) 177 | } 178 | if err != nil { 179 | return err 180 | } 181 | err = env.Open(path, lmdbcmd.OpenFlag(), 644) 182 | defer env.Close() 183 | if err != nil { 184 | return err 185 | } 186 | return env.View(func(txn *lmdb.Txn) (err error) { 187 | if opt == nil || len(opt.DB) == 0 { 188 | err := catRoot(txn) 189 | if err != nil { 190 | return err 191 | } 192 | } 193 | if opt != nil { 194 | for _, dbname := range opt.DB { 195 | err := catDB(txn, dbname) 196 | if err != nil { 197 | return fmt.Errorf("%v (%q)", err, dbname) 198 | } 199 | } 200 | } 201 | return nil 202 | }) 203 | } 204 | 205 | func catDB(txn *lmdb.Txn, dbname string) error { 206 | dbi, err := txn.OpenDBI(dbname, 0) 207 | if err != nil { 208 | return err 209 | } 210 | return catDBI(txn, dbi) 211 | } 212 | 213 | func catRoot(txn *lmdb.Txn) error { 214 | dbi, err := txn.OpenRoot(0) 215 | if err != nil { 216 | return err 217 | } 218 | return catDBI(txn, dbi) 219 | } 220 | 221 | func catDBI(txn *lmdb.Txn, dbi lmdb.DBI) error { 222 | s := lmdbscan.New(txn, dbi) 223 | defer s.Close() 224 | for s.Scan() { 225 | fmt.Println(string(s.Val())) 226 | } 227 | return s.Err() 228 | } 229 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/exp/lmdbscan/example_test.go: -------------------------------------------------------------------------------- 1 | package lmdbscan_test 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | 7 | "github.com/bmatsuo/lmdb-go/exp/lmdbscan" 8 | "github.com/bmatsuo/lmdb-go/lmdb" 9 | ) 10 | 11 | var env *lmdb.Env 12 | var dbi lmdb.DBI 13 | 14 | // This example demonstrates basic usage of a Scanner to scan a database. It 15 | // is important to always call scanner.Err() which will returned any unexpected 16 | // error which interrupted scanner.Scan(). 17 | func ExampleScanner() { 18 | err := env.View(func(txn *lmdb.Txn) (err error) { 19 | scanner := lmdbscan.New(txn, dbi) 20 | defer scanner.Close() 21 | 22 | for scanner.Scan() { 23 | log.Printf("k=%q v=%q", scanner.Key(), scanner.Val()) 24 | } 25 | return scanner.Err() 26 | }) 27 | if err != nil { 28 | panic(err) 29 | } 30 | } 31 | 32 | // This example demonstrates scanning a key range in a database. Set is used 33 | // to move the cursor's starting position to the desired prefix. 34 | func ExampleScanner_Set() { 35 | keyprefix := []byte("users:") 36 | err := env.View(func(txn *lmdb.Txn) (err error) { 37 | scanner := lmdbscan.New(txn, dbi) 38 | defer scanner.Close() 39 | 40 | scanner.Set(keyprefix, nil, lmdb.SetRange) 41 | for scanner.Scan() { 42 | if !bytes.HasPrefix(scanner.Key(), keyprefix) { 43 | break 44 | } 45 | log.Printf("k=%q v=%q", scanner.Key(), scanner.Val()) 46 | } 47 | return scanner.Err() 48 | }) 49 | if err != nil { 50 | panic(err) 51 | } 52 | } 53 | 54 | // This example demonstrates scanning all values for a key in a root database 55 | // with the lmdb.DupSort flag set. SetNext is used instead of Set to configure 56 | // Cursor the to return ErrNotFound (EOF) after all duplicate keys have been 57 | // iterated. 58 | func ExampleScanner_SetNext() { 59 | key := []byte("userphone:123") 60 | err := env.View(func(txn *lmdb.Txn) (err error) { 61 | scanner := lmdbscan.New(txn, dbi) 62 | defer scanner.Close() 63 | 64 | scanner.SetNext(key, nil, lmdb.GetBothRange, lmdb.NextDup) 65 | for scanner.Scan() { 66 | log.Printf("k=%q v=%q", scanner.Key(), scanner.Val()) 67 | } 68 | return scanner.Err() 69 | }) 70 | if err != nil { 71 | panic(err) 72 | } 73 | } 74 | 75 | // This example demonstrates scanning all values for duplicate keys in a 76 | // database with the lmdb.DupSort flag set. Two loops are used to iterate over 77 | // unique keys and their values respectively. The example exploits the return 78 | // value from SetNext as the termination condition for the first loop. 79 | func ExampleScanner_SetNext_nextNoDup() { 80 | err := env.View(func(txn *lmdb.Txn) (err error) { 81 | scanner := lmdbscan.New(txn, dbi) 82 | defer scanner.Close() 83 | 84 | for scanner.SetNext(nil, nil, lmdb.NextNoDup, lmdb.NextDup) { 85 | key := scanner.Key() 86 | var vals [][]byte 87 | for scanner.Scan() { 88 | vals = append(vals, scanner.Val()) 89 | } 90 | log.Printf("k=%q v=%q", key, vals) 91 | if scanner.Err() != nil { 92 | break 93 | } 94 | } 95 | return scanner.Err() 96 | }) 97 | if err != nil { 98 | panic(err) 99 | } 100 | } 101 | 102 | // This advanced example demonstrates batch scanning of values for duplicate 103 | // keys in a database with the lmdb.DupFixed and lmdb.DupSort flags set. The 104 | // outer loop scans unique keys and the inner loop scans duplicate values. The 105 | // GetMultiple op requires an additional check following its use to determine 106 | // if no duplicates exist in the database. 107 | func ExampleScanner_SetNext_getMultiple() { 108 | err := env.View(func(txn *lmdb.Txn) (err error) { 109 | scanner := lmdbscan.New(txn, dbi) 110 | defer scanner.Close() 111 | 112 | for scanner.Set(nil, nil, lmdb.NextNoDup) { 113 | key := scanner.Key() 114 | valFirst := scanner.Val() 115 | var vals [][]byte 116 | if !scanner.SetNext(nil, nil, lmdb.GetMultiple, lmdb.NextMultiple) { 117 | // only one value exists for the key, and it has been scanned. 118 | vals = append(vals, valFirst) 119 | } 120 | for scanner.Scan() { 121 | // this loop is only entered if multiple values exist for key. 122 | multi := lmdb.WrapMulti(scanner.Val(), len(valFirst)) 123 | vals = append(vals, multi.Vals()...) 124 | } 125 | log.Printf("k=%q v=%q", key, vals) 126 | if scanner.Err() != nil { 127 | break 128 | } 129 | } 130 | return scanner.Err() 131 | }) 132 | if err != nil { 133 | panic(err) 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/exp/lmdbscan/scanner.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package lmdbscan provides a wrapper for lmdb.Cursor to simplify iteration. 3 | This package is experimental and it's API may change. 4 | */ 5 | package lmdbscan 6 | 7 | import ( 8 | "fmt" 9 | 10 | "github.com/bmatsuo/lmdb-go/lmdb" 11 | ) 12 | 13 | // errClosed is an error returned to the user when attempting to operate on a 14 | // closed Scanner. 15 | var errClosed = fmt.Errorf("scanner is closed") 16 | 17 | // Scanner is a low level construct for scanning databases inside a 18 | // transaction. 19 | type Scanner struct { 20 | dbi lmdb.DBI 21 | cur *lmdb.Cursor 22 | op uint 23 | key []byte 24 | val []byte 25 | err error 26 | set bool 27 | } 28 | 29 | // New allocates and intializes a Scanner for dbi within txn. When the Scanner 30 | // returned by New is no longer needed its Close method must be called. 31 | func New(txn *lmdb.Txn, dbi lmdb.DBI) *Scanner { 32 | s := &Scanner{ 33 | dbi: dbi, 34 | op: lmdb.Next, 35 | } 36 | 37 | s.cur, s.err = txn.OpenCursor(dbi) 38 | return s 39 | } 40 | 41 | // Cursor returns the lmdb.Cursor underlying s. Cursor returns nil if s is 42 | // closed. 43 | func (s *Scanner) Cursor() *lmdb.Cursor { 44 | return s.cur 45 | } 46 | 47 | // Del will delete the key at the current cursor location. 48 | // 49 | // Del is deprecated. Instead use s.Cursor().Del(flags). 50 | func (s *Scanner) Del(flags uint) error { 51 | if s.cur == nil { 52 | return errClosed 53 | } 54 | return s.cur.Del(flags) 55 | } 56 | 57 | // Key returns the key read during the last call to Scan. 58 | func (s *Scanner) Key() []byte { 59 | return s.key 60 | } 61 | 62 | // Val returns the value read during the last call to Scan. 63 | func (s *Scanner) Val() []byte { 64 | return s.val 65 | } 66 | 67 | // Set moves the cursor with s.Cursor().Get(k, v, opset), and sets s.Key(), 68 | // s.Val(), and s.Err() accordingly. The cursor will not move in the next call 69 | // to Scan. 70 | func (s *Scanner) Set(k, v []byte, opset uint) bool { 71 | if !s.checkOpen() { 72 | return false 73 | } 74 | s.set = true 75 | s.key, s.val, s.err = s.cur.Get(k, v, opset) 76 | return s.err == nil 77 | } 78 | 79 | // SetNext moves the cursor like s.Set(k, v, opset) for the next call to 80 | // s.Scan(). Subsequent calls to s.Scan() move the cursor as c.Get(nil, nil, 81 | // opnext) 82 | func (s *Scanner) SetNext(k, v []byte, opset, opnext uint) bool { 83 | if !s.checkOpen() { 84 | return false 85 | } 86 | ok := s.Set(k, v, opset) 87 | s.op = opnext 88 | return ok 89 | } 90 | 91 | // Scan gets successive key-value pairs using the underlying cursor. Scan 92 | // returns false when key-value pairs are exhausted or another error is 93 | // encountered. 94 | func (s *Scanner) Scan() bool { 95 | if !s.checkOpen() { 96 | return false 97 | } 98 | if s.set { 99 | s.set = false 100 | } else { 101 | s.key, s.val, s.err = s.cur.Get(nil, nil, s.op) 102 | } 103 | return s.err == nil 104 | } 105 | 106 | func (s *Scanner) checkOpen() bool { 107 | if s.cur != nil { 108 | return true 109 | } 110 | if s.err == nil { 111 | s.err = errClosed 112 | } 113 | return false 114 | } 115 | 116 | // Err returns a non-nil error if and only if the previous call to s.Scan() 117 | // resulted in an error other than lmdb.ErrNotFound. 118 | func (s *Scanner) Err() error { 119 | if lmdb.IsNotFound(s.err) { 120 | return nil 121 | } 122 | return s.err 123 | } 124 | 125 | // Close closes the cursor underlying s and clears its ows internal structures. 126 | // Close does not attempt to terminate the enclosing transaction. 127 | // 128 | // Scan must not be called after Close. 129 | func (s *Scanner) Close() { 130 | if s.cur != nil { 131 | s.cur.Close() 132 | s.cur = nil 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/exp/lmdbscan/scanner_test.go: -------------------------------------------------------------------------------- 1 | package lmdbscan 2 | 3 | import ( 4 | "reflect" 5 | "syscall" 6 | "testing" 7 | 8 | "github.com/bmatsuo/lmdb-go/internal/lmdbtest" 9 | "github.com/bmatsuo/lmdb-go/lmdb" 10 | ) 11 | 12 | type errcheck func(err error) (ok bool) 13 | 14 | var pIsNil = func(err error) bool { return err == nil } 15 | 16 | func TestScanner_err(t *testing.T) { 17 | env, err := lmdbtest.NewEnv(nil) 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | defer lmdbtest.Destroy(env) 22 | 23 | err = env.View(func(txn *lmdb.Txn) (err error) { 24 | scanner := New(txn, 123) 25 | defer scanner.Close() 26 | for scanner.Scan() { 27 | t.Error("loop should not execute") 28 | } 29 | if scanner.Set(nil, nil, lmdb.First) { 30 | t.Error("Set returned true") 31 | } 32 | if scanner.SetNext(nil, nil, lmdb.NextNoDup, lmdb.NextDup) { 33 | t.Error("SetNext returned true") 34 | } 35 | return scanner.Err() 36 | }) 37 | if !lmdb.IsErrnoSys(err, syscall.EINVAL) { 38 | t.Errorf("unexpected error: %q (!= %q)", err, syscall.EINVAL) 39 | } 40 | } 41 | 42 | func TestScanner_closed(t *testing.T) { 43 | env, err := lmdbtest.NewEnv(nil) 44 | if err != nil { 45 | t.Fatal(err) 46 | } 47 | defer lmdbtest.Destroy(env) 48 | 49 | err = env.View(func(txn *lmdb.Txn) (err error) { 50 | dbi, err := txn.OpenRoot(0) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | scanner := New(txn, dbi) 56 | 57 | err = scanner.Err() 58 | if err != nil { 59 | return err 60 | } 61 | 62 | scanner.Close() 63 | 64 | for scanner.Scan() { 65 | t.Error("loop should not execute") 66 | } 67 | return scanner.Err() 68 | }) 69 | if err != errClosed { 70 | t.Errorf("unexpected error: %q (!= %q)", err, errClosed) 71 | } 72 | } 73 | 74 | func TestScanner_Scan(t *testing.T) { 75 | env, err := lmdbtest.NewEnv(nil) 76 | if err != nil { 77 | t.Fatal(err) 78 | } 79 | defer lmdbtest.Destroy(env) 80 | 81 | dbi, err := lmdbtest.OpenRoot(env, 0) 82 | if err != nil { 83 | t.Error(err) 84 | return 85 | } 86 | 87 | items := lmdbtest.SimpleItemList{ 88 | {"k0", "v0"}, 89 | {"k1", "v1"}, 90 | {"k2", "v2"}, 91 | {"k3", "v3"}, 92 | {"k4", "v4"}, 93 | {"k5", "v5"}, 94 | } 95 | err = lmdbtest.Put(env, dbi, items) 96 | if err != nil { 97 | t.Error(err) 98 | } 99 | scanned, err := simplescan(env, dbi) 100 | if err != nil { 101 | t.Errorf("%v", err) 102 | } 103 | if !reflect.DeepEqual(scanned, items) { 104 | t.Errorf("unexpected items %q (!= %q)", scanned, items) 105 | } 106 | } 107 | 108 | func TestScanner_Set(t *testing.T) { 109 | env, err := lmdbtest.NewEnv(nil) 110 | if err != nil { 111 | t.Fatal(err) 112 | } 113 | defer lmdbtest.Destroy(env) 114 | 115 | dbi, err := lmdbtest.OpenRoot(env, 0) 116 | if err != nil { 117 | t.Error(err) 118 | return 119 | } 120 | 121 | items := lmdbtest.SimpleItemList{ 122 | {"k0", "v0"}, 123 | {"k1", "v1"}, 124 | {"k2", "v2"}, 125 | {"k3", "v3"}, 126 | {"k4", "v4"}, 127 | {"k5", "v5"}, 128 | } 129 | err = lmdbtest.Put(env, dbi, items) 130 | if err != nil { 131 | t.Error(err) 132 | } 133 | 134 | var tail lmdbtest.SimpleItemList 135 | err = env.View(func(txn *lmdb.Txn) (err error) { 136 | dbi, err := txn.OpenRoot(0) 137 | if err != nil { 138 | return err 139 | } 140 | s := New(txn, dbi) 141 | defer s.Close() 142 | 143 | s.Set([]byte("k34"), nil, lmdb.SetRange) 144 | tail, err = remaining(s) 145 | return err 146 | }) 147 | if err != nil { 148 | t.Error(err) 149 | } 150 | if !reflect.DeepEqual(tail, items[4:]) { 151 | t.Errorf("items: %q (!= %q)", tail, items) 152 | } 153 | } 154 | 155 | func TestScanner_SetNext(t *testing.T) { 156 | env, err := lmdbtest.NewEnv(nil) 157 | if err != nil { 158 | t.Fatal(err) 159 | } 160 | defer lmdbtest.Destroy(env) 161 | 162 | dbi, err := lmdbtest.OpenRoot(env, 0) 163 | if err != nil { 164 | t.Error(err) 165 | return 166 | } 167 | 168 | items := lmdbtest.SimpleItemList{ 169 | {"k0", "v0"}, 170 | {"k1", "v1"}, 171 | {"k2", "v2"}, 172 | {"k3", "v3"}, 173 | {"k4", "v4"}, 174 | {"k5", "v5"}, 175 | } 176 | err = lmdbtest.Put(env, dbi, items) 177 | if err != nil { 178 | t.Error(err) 179 | } 180 | 181 | var head lmdbtest.SimpleItemList 182 | err = env.View(func(txn *lmdb.Txn) (err error) { 183 | dbi, err := txn.OpenRoot(0) 184 | if err != nil { 185 | return err 186 | } 187 | s := New(txn, dbi) 188 | defer s.Close() 189 | 190 | s.SetNext([]byte("k34"), nil, lmdb.SetRange, lmdb.Prev) 191 | head, err = remaining(s) 192 | return err 193 | }) 194 | if err != nil { 195 | t.Error(err) 196 | } 197 | 198 | // reverse head before testing its value 199 | n := len(head) 200 | for i := 0; i < n/2; i++ { 201 | head[i], head[n-1-i] = head[n-1-i], head[i] 202 | } 203 | 204 | if !reflect.DeepEqual(head, items[:5]) { 205 | t.Errorf("items: %q (!= %q)", head, items) 206 | } 207 | } 208 | 209 | func TestScanner_Del(t *testing.T) { 210 | env, err := lmdbtest.NewEnv(nil) 211 | if err != nil { 212 | t.Fatal(err) 213 | } 214 | defer lmdbtest.Destroy(env) 215 | 216 | dbi, err := lmdbtest.OpenRoot(env, 0) 217 | if err != nil { 218 | t.Error(err) 219 | return 220 | } 221 | 222 | items := lmdbtest.SimpleItemList{ 223 | {"k0", "v0"}, 224 | {"k1", "v1"}, 225 | {"k2", "v2"}, 226 | {"k3", "v3"}, 227 | {"k4", "v4"}, 228 | {"k5", "v5"}, 229 | } 230 | err = lmdbtest.Put(env, dbi, items) 231 | if err != nil { 232 | t.Error(err) 233 | } 234 | 235 | err = env.Update(func(txn *lmdb.Txn) (err error) { 236 | s := New(txn, dbi) 237 | defer s.Close() 238 | for s.Scan() { 239 | err = s.Del(0) 240 | if err != nil { 241 | return err 242 | } 243 | } 244 | return s.Err() 245 | }) 246 | if err != nil { 247 | t.Error(err) 248 | } 249 | 250 | var rem lmdbtest.SimpleItemList 251 | err = env.View(func(txn *lmdb.Txn) (err error) { 252 | s := New(txn, dbi) 253 | defer s.Close() 254 | rem, err = remaining(s) 255 | return err 256 | }) 257 | if err != nil { 258 | t.Error(err) 259 | } 260 | 261 | if len(rem) != 0 { 262 | t.Errorf("items: %q (!= %q)", rem, []string{}) 263 | } 264 | } 265 | 266 | func TestScanner_Del_closed(t *testing.T) { 267 | env, err := lmdbtest.NewEnv(nil) 268 | if err != nil { 269 | t.Fatal(err) 270 | } 271 | defer lmdbtest.Destroy(env) 272 | 273 | dbi, err := lmdbtest.OpenRoot(env, 0) 274 | if err != nil { 275 | t.Error(err) 276 | return 277 | } 278 | 279 | items := lmdbtest.SimpleItemList{ 280 | {"k0", "v0"}, 281 | {"k1", "v1"}, 282 | {"k2", "v2"}, 283 | {"k3", "v3"}, 284 | {"k4", "v4"}, 285 | {"k5", "v5"}, 286 | } 287 | err = lmdbtest.Put(env, dbi, items) 288 | if err != nil { 289 | t.Error(err) 290 | } 291 | 292 | err = env.Update(func(txn *lmdb.Txn) (err error) { 293 | s := New(txn, dbi) 294 | s.Close() 295 | return s.Del(0) 296 | }) 297 | if err != errClosed { 298 | t.Errorf("unexpected error: %q (!= %q)", err, errClosed) 299 | } 300 | } 301 | 302 | func TestScanner_Cursor_Del(t *testing.T) { 303 | env, err := lmdbtest.NewEnv(nil) 304 | if err != nil { 305 | t.Fatal(err) 306 | } 307 | defer lmdbtest.Destroy(env) 308 | 309 | dbi, err := lmdbtest.OpenRoot(env, 0) 310 | if err != nil { 311 | t.Error(err) 312 | return 313 | } 314 | 315 | items := lmdbtest.SimpleItemList{ 316 | {"k0", "v0"}, 317 | {"k1", "v1"}, 318 | {"k2", "v2"}, 319 | {"k3", "v3"}, 320 | {"k4", "v4"}, 321 | {"k5", "v5"}, 322 | } 323 | err = lmdbtest.Put(env, dbi, items) 324 | if err != nil { 325 | t.Error(err) 326 | } 327 | 328 | err = env.Update(func(txn *lmdb.Txn) (err error) { 329 | s := New(txn, dbi) 330 | defer s.Close() 331 | cur := s.Cursor() 332 | for s.Scan() { 333 | err = cur.Del(0) 334 | if err != nil { 335 | return err 336 | } 337 | } 338 | return s.Err() 339 | }) 340 | if err != nil { 341 | t.Error(err) 342 | } 343 | 344 | var rem lmdbtest.SimpleItemList 345 | err = env.View(func(txn *lmdb.Txn) (err error) { 346 | s := New(txn, dbi) 347 | defer s.Close() 348 | rem, err = remaining(s) 349 | return err 350 | }) 351 | if err != nil { 352 | t.Error(err) 353 | } 354 | 355 | if len(rem) != 0 { 356 | t.Errorf("items: %q (!= %q)", rem, []string{}) 357 | } 358 | } 359 | 360 | func simplescan(env *lmdb.Env, dbi lmdb.DBI) (items lmdbtest.SimpleItemList, err error) { 361 | err = env.View(func(txn *lmdb.Txn) (err error) { 362 | s := New(txn, dbi) 363 | defer s.Close() 364 | 365 | items, err = remaining(s) 366 | return err 367 | }) 368 | return items, err 369 | } 370 | 371 | func remaining(s *Scanner) (items lmdbtest.SimpleItemList, err error) { 372 | for s.Scan() { 373 | item := &lmdbtest.SimpleItem{ 374 | K: string(s.Key()), 375 | V: string(s.Val()), 376 | } 377 | items = append(items, item) 378 | } 379 | err = s.Err() 380 | if err != nil { 381 | return nil, err 382 | } 383 | return items, nil 384 | } 385 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/exp/lmdbsync/handler.go: -------------------------------------------------------------------------------- 1 | package lmdbsync 2 | 3 | import ( 4 | "errors" 5 | "math" 6 | "math/rand" 7 | "time" 8 | 9 | "golang.org/x/net/context" 10 | 11 | "github.com/bmatsuo/lmdb-go/lmdb" 12 | ) 13 | 14 | // Handler can intercept errors returned by a transaction and handle them in an 15 | // application-specific way, including by resizing the environment and retrying 16 | // the transaction by returning ErrTxnRetry. 17 | type Handler interface { 18 | HandleTxnErr(ctx context.Context, env *Env, err error) (context.Context, error) 19 | } 20 | 21 | // HandlerChain is a Handler implementation that iteratively calls each handler 22 | // in the underlying slice when handling an error. 23 | type HandlerChain []Handler 24 | 25 | // HandleTxnErr implements the Handler interface. Each handler in c processes 26 | // the context.Context and error returned by the previous handler. 27 | func (c HandlerChain) HandleTxnErr(ctx context.Context, env *Env, err error) (context.Context, error) { 28 | for _, h := range c { 29 | ctx, err = h.HandleTxnErr(ctx, env, err) 30 | } 31 | return ctx, err 32 | } 33 | 34 | // Append returns a new HandlerChain that will evaluate h in sequence after the 35 | // Handlers already in C are evaluated. Append does not modify the storage 36 | // undelying c. 37 | func (c HandlerChain) Append(h ...Handler) HandlerChain { 38 | _c := make(HandlerChain, len(c)+len(h)) 39 | copy(_c, c) 40 | copy(_c[len(c):], h) 41 | return _c 42 | } 43 | 44 | // MapResizedHandler returns a Handler that transparently adopts map sizes set 45 | // by external processes and retries any transactions that failed to start 46 | // because of lmdb.MapResized. 47 | // 48 | // If the database is growing too rapidly and maxRetry consecutive transactions 49 | // fail due to lmdb.MapResized then the Handler returned by MapResizedHandler 50 | // gives up and returns the lmdb.MapResized error to the caller. Delay will be 51 | // called before each call to Env.SetMapSize to insert an optional delay. 52 | // 53 | // Open transactions must not directly create new (non-child) transactions when 54 | // using MapResizedHandler or the environment will deadlock. 55 | func MapResizedHandler(maxRetry int, delay DelayFunc) Handler { 56 | if maxRetry == 0 { 57 | maxRetry = MapResizedDefaultRetry 58 | } 59 | if delay == nil { 60 | delay = MapResizedDefaultDelay 61 | } 62 | return &resizedHandler{ 63 | MaxRetry: maxRetry, 64 | Delay: delay, 65 | } 66 | } 67 | 68 | // MapResizedDefaultRetry is the default number of attempts MapResizedHandler 69 | // will make adopt a new map size when lmdb.MapResized is encountered 70 | // repeatedly. 71 | var MapResizedDefaultRetry = 2 72 | 73 | // DelayFunc takes as input the number of previous attempts and returns the 74 | // delay before making another attempt. 75 | type DelayFunc func(attempt int) time.Duration 76 | 77 | // ExponentialBackoff returns a function that delays each attempt by random 78 | // number between 0 and the minimum of max and base*factor^attempt. 79 | func ExponentialBackoff(base time.Duration, max time.Duration, factor float64) DelayFunc { 80 | return func(attempt int) time.Duration { 81 | _max := float64(base) * math.Pow(factor, float64(attempt)) 82 | _max = math.Min(_max, float64(max)) 83 | n := rand.Int63n(int64(_max)) 84 | return time.Duration(n) 85 | } 86 | } 87 | 88 | // MapResizedDefaultDelay is the default DelayFunc when MapResizedHandler is 89 | // passed a nil value. 90 | var MapResizedDefaultDelay = ExponentialBackoff(time.Millisecond, 5*time.Millisecond, 2) 91 | 92 | // MapFullFunc is a function for resizing a memory map after it has become 93 | // full. The function receives the current map size as its argument and 94 | // returns a new map size. The new size will only be applied if the second 95 | // return value is true. 96 | type MapFullFunc func(size int64) (int64, bool) 97 | 98 | // MapFullHandler returns a Handler that retries updates which failed to commit 99 | // due to lmdb.MapFull errors. When lmdb.MapFull is encountered fn is used to 100 | // set a new new map size before opening a new transaction and executing the 101 | // lmdb.TxnOp again. 102 | // 103 | // When using MapFullHandler it is important that updates are idempotent. An 104 | // Env.Update that encounters lmdb.MapFull may execute its lmdb.TxnOp function 105 | // multiple times before successfully committing it (or aborting). 106 | // 107 | // Open view transactions must not wait for updates to complete when using 108 | // MapFullHandler or the environment will deadlock. 109 | func MapFullHandler(fn MapFullFunc) Handler { 110 | return &mapFullHandler{fn} 111 | } 112 | 113 | // ErrTxnRetry is returned by a Handler to have the Env retry the transaction. 114 | var ErrTxnRetry = errors.New("lmdbsync: retry failed txn") 115 | 116 | // TxnRunner is an interface for types that can run lmdb transactions. 117 | // TxnRunner is satisfied by Env. 118 | type TxnRunner interface { 119 | RunTxn(flags uint, op lmdb.TxnOp) error 120 | View(op lmdb.TxnOp) error 121 | Update(op lmdb.TxnOp) error 122 | UpdateLocked(op lmdb.TxnOp) error 123 | WithHandler(h Handler) TxnRunner 124 | } 125 | 126 | type handlerRunner struct { 127 | env *Env 128 | h Handler 129 | } 130 | 131 | func (r *handlerRunner) WithHandler(h Handler) TxnRunner { 132 | return &handlerRunner{ 133 | env: r.env, 134 | h: HandlerChain{r.h, h}, 135 | } 136 | } 137 | 138 | func (r *handlerRunner) RunTxn(flags uint, op lmdb.TxnOp) error { 139 | readonly := flags&lmdb.Readonly != 0 140 | return r.env.runHandler(readonly, func() error { return r.env.RunTxn(flags, op) }, r.h) 141 | } 142 | 143 | func (r *handlerRunner) View(op lmdb.TxnOp) error { 144 | return r.env.runHandler(true, func() error { return r.env.View(op) }, r.h) 145 | } 146 | 147 | func (r *handlerRunner) Update(op lmdb.TxnOp) error { 148 | return r.env.runHandler(false, func() error { return r.env.Update(op) }, r.h) 149 | } 150 | 151 | func (r *handlerRunner) UpdateLocked(op lmdb.TxnOp) error { 152 | return r.env.runHandler(false, func() error { return r.env.UpdateLocked(op) }, r.h) 153 | } 154 | 155 | type mapFullHandler struct { 156 | fn MapFullFunc 157 | } 158 | 159 | func (h *mapFullHandler) HandleTxnErr(ctx context.Context, env *Env, err error) (context.Context, error) { 160 | if !lmdb.IsMapFull(err) { 161 | return ctx, err 162 | } 163 | 164 | newsize, ok := h.getNewSize(env) 165 | if !ok { 166 | return ctx, err 167 | } 168 | if env.setMapSize(newsize, 0) != nil { 169 | return ctx, err 170 | } 171 | 172 | return ctx, ErrTxnRetry 173 | } 174 | 175 | func (h *mapFullHandler) getNewSize(env *Env) (int64, bool) { 176 | info, err := env.Info() 177 | if err != nil { 178 | return 0, false 179 | } 180 | newsize, ok := h.fn(info.MapSize) 181 | if !ok || newsize <= info.MapSize { 182 | return 0, false 183 | } 184 | return newsize, true 185 | } 186 | 187 | type resizedHandlerKey int 188 | 189 | type resizeRetryCount struct { 190 | n int 191 | } 192 | 193 | func (r *resizeRetryCount) Get() int { 194 | if r == nil { 195 | return 0 196 | } 197 | return r.n 198 | } 199 | 200 | func (r *resizeRetryCount) Add(n int) *resizeRetryCount { 201 | if r == nil { 202 | return &resizeRetryCount{1} 203 | } 204 | return &resizeRetryCount{r.n + 1} 205 | } 206 | 207 | func getResizedRetryCount(ctx context.Context) *resizeRetryCount { 208 | v, _ := ctx.Value(resizedHandlerKey(0)).(*resizeRetryCount) 209 | return v 210 | } 211 | 212 | func withResizedRetryCount(ctx context.Context, count *resizeRetryCount) context.Context { 213 | return context.WithValue(ctx, resizedHandlerKey(0), count) 214 | } 215 | 216 | type resizedHandler struct { 217 | MaxRetry int 218 | Delay DelayFunc 219 | } 220 | 221 | func (h *resizedHandler) HandleTxnErr(ctx context.Context, env *Env, err error) (context.Context, error) { 222 | if !lmdb.IsMapResized(err) { 223 | ctx := context.WithValue(ctx, resizedHandlerKey(0), nil) 224 | return ctx, err 225 | } 226 | 227 | count := getResizedRetryCount(ctx) 228 | numRetry := count.Get() 229 | 230 | // fail the transaction with MapResized error when too many attempts have 231 | // been made. 232 | maxRetry := h.MaxRetry 233 | if maxRetry > 0 && numRetry >= maxRetry { 234 | ctx := withResizedRetryCount(ctx, nil) 235 | return ctx, err 236 | } 237 | 238 | ctx = withResizedRetryCount(ctx, count.Add(1)) 239 | 240 | delay := h.Delay(numRetry) 241 | err = env.setMapSize(0, delay) 242 | if err != nil { 243 | return ctx, err 244 | } 245 | return ctx, ErrTxnRetry 246 | } 247 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/exp/lmdbsync/handler_test.go: -------------------------------------------------------------------------------- 1 | package lmdbsync 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | "golang.org/x/net/context" 9 | 10 | "github.com/bmatsuo/lmdb-go/internal/lmdbtest" 11 | "github.com/bmatsuo/lmdb-go/lmdb" 12 | ) 13 | 14 | type testHandler struct { 15 | called bool 16 | ctx context.Context 17 | env *Env 18 | err error 19 | } 20 | 21 | func (h *testHandler) HandleTxnErr(ctx context.Context, env *Env, err error) (context.Context, error) { 22 | h.called = true 23 | h.ctx = ctx 24 | h.env = env 25 | h.err = err 26 | return ctx, err 27 | } 28 | 29 | func TestHandlerChain(t *testing.T) { 30 | ctx := context.Background() 31 | env, err := newEnv(nil) 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | defer lmdbtest.Destroy(env.Env) 36 | 37 | var chain1 HandlerChain 38 | errother := fmt.Errorf("testerr") 39 | 40 | ctx1, err := chain1.HandleTxnErr(ctx, env, errother) 41 | if err != errother { 42 | t.Error(err) 43 | } 44 | if ctx1 != ctx { 45 | t.Errorf("unexpected ctx: %#v (!= %#v)", ctx1, ctx) 46 | } 47 | 48 | chain2 := chain1.Append(&passthroughHandler{}) 49 | ctx2, err := chain2.HandleTxnErr(ctx, env, errother) 50 | if err != errother { 51 | t.Error(err) 52 | } 53 | if ctx2 != ctx { 54 | t.Errorf("unexpected ctx: %#v (!= %#v)", ctx2, ctx) 55 | } 56 | } 57 | 58 | type retryHandler struct{} 59 | 60 | func (*retryHandler) HandleTxnErr(ctx context.Context, env *Env, err error) (context.Context, error) { 61 | return ctx, ErrTxnRetry 62 | 63 | } 64 | 65 | type passthroughHandler struct{} 66 | 67 | func (*passthroughHandler) HandleTxnErr(ctx context.Context, env *Env, err error) (context.Context, error) { 68 | return ctx, err 69 | 70 | } 71 | 72 | func TestMapFullHandler(t *testing.T) { 73 | ctx := context.Background() 74 | env, err := newEnv(nil) 75 | if err != nil { 76 | t.Fatal(err) 77 | } 78 | defer lmdbtest.Destroy(env.Env) 79 | 80 | info, err := env.Info() 81 | if err != nil { 82 | t.Error(err) 83 | return 84 | } 85 | orig := info.MapSize 86 | 87 | doubleSize := func(size int64) (int64, bool) { return size * 2, true } 88 | handler := MapFullHandler(doubleSize) 89 | 90 | errother := fmt.Errorf("testerr") 91 | ctx1, err := handler.HandleTxnErr(ctx, env, errother) 92 | if ctx1 != ctx { 93 | t.Errorf("ctx changed: %q (!= %q)", ctx1, ctx) 94 | } 95 | 96 | errmapfull := &lmdb.OpError{ 97 | Op: "lmdbsync_test_op", 98 | Errno: lmdb.MapFull, 99 | } 100 | ctx1, err = handler.HandleTxnErr(ctx, env, errmapfull) 101 | if err != ErrTxnRetry { 102 | t.Errorf("unexpected error: %v", err) 103 | } 104 | if ctx1 != ctx { 105 | t.Errorf("ctx changed: %q (!= %q)", ctx1, ctx) 106 | } 107 | 108 | info, err = env.Info() 109 | if err != nil { 110 | t.Error(err) 111 | return 112 | } 113 | if info.MapSize <= orig { 114 | t.Errorf("unexpected map size: %d (<= %d)", info.MapSize, orig) 115 | } 116 | } 117 | 118 | func TestMapResizedHandler(t *testing.T) { 119 | ctx := context.Background() 120 | env, err := newEnv(nil) 121 | if err != nil { 122 | t.Fatal(err) 123 | } 124 | defer lmdbtest.Destroy(env.Env) 125 | 126 | handler := MapResizedHandler(2, func(int) time.Duration { return 100 * time.Microsecond }) 127 | 128 | errother := fmt.Errorf("testerr") 129 | _, err = handler.HandleTxnErr(ctx, env, errother) 130 | 131 | errmapresized := &lmdb.OpError{ 132 | Op: "lmdbsync_test_op", 133 | Errno: lmdb.MapResized, 134 | } 135 | ctx1, err := handler.HandleTxnErr(ctx, env, errmapresized) 136 | if err != ErrTxnRetry { 137 | t.Errorf("unexpected error: %v", err) 138 | } 139 | ctx2, err := handler.HandleTxnErr(ctx1, env, errmapresized) 140 | if err != ErrTxnRetry { 141 | t.Errorf("unexpected error: %v", err) 142 | } 143 | 144 | // after MapResized has been encountered enough times consecutively the 145 | // handler starts passing MapResized through to the caller. 146 | _, err = handler.HandleTxnErr(ctx2, env, errmapresized) 147 | if !lmdb.IsMapResized(err) { 148 | t.Errorf("unexpected error: %v", err) 149 | } 150 | ctx3, err := handler.HandleTxnErr(ctx2, env, errmapresized) 151 | if !lmdb.IsMapResized(err) { 152 | t.Errorf("unexpected error: %v", err) 153 | } 154 | 155 | ctx4, err := handler.HandleTxnErr(ctx3, env, errother) 156 | if err != errother { 157 | t.Errorf("unexpected error: %v", err) 158 | } 159 | 160 | // after encountering an error other than MapResized the handler resets its 161 | // failure count and will continue attempting to adopt the new map size 162 | // when MapResized is encountered. 163 | ctx5, err := handler.HandleTxnErr(ctx4, env, errmapresized) 164 | if err != ErrTxnRetry { 165 | t.Errorf("unexpected error: %v", err) 166 | } 167 | _, err = handler.HandleTxnErr(ctx5, env, errmapresized) 168 | if err != ErrTxnRetry { 169 | t.Errorf("unexpected error: %v", err) 170 | } 171 | } 172 | 173 | func TestExponentialBackoff(t *testing.T) { 174 | base := time.Millisecond 175 | max := 3 * time.Millisecond 176 | factor := 2.0 177 | backoff := ExponentialBackoff(base, max, factor) 178 | 179 | const numtest = 100 180 | for i := 0; i < numtest; i++ { 181 | n := backoff(0) 182 | if n < 0 || n > base { 183 | t.Errorf("unexpected backoff: %v", n) 184 | } 185 | } 186 | for i := 0; i < numtest; i++ { 187 | n := backoff(1) 188 | if n < 0 || n > 2*base { 189 | t.Errorf("unexpected backoff: %v", n) 190 | } 191 | } 192 | for i := 0; i < numtest; i++ { 193 | n := backoff(2) 194 | if n < 0 || n > max { 195 | t.Errorf("unexpected backoff: %v", n) 196 | } 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/exp/lmdbsync/lmdbsync.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package lmdbsync provides advanced synchronization for LMDB environments at the 3 | cost of performance. The package provides a drop-in replacement for *lmdb.Env 4 | that can be used in situations where the database may be resized or where the 5 | flag lmdb.NoLock is used. 6 | 7 | Bypassing an Env's methods to access the underlying lmdb.Env is not safe. The 8 | severity of such usage depends such behavior should be strictly avoided as it 9 | may produce undefined behavior from the LMDB C library. 10 | 11 | Resizing the environment 12 | 13 | The Env type synchronizes all calls to Env.SetMapSize so that it may, with some 14 | caveats, be safely called in the presence of concurrent transactions after an 15 | environment has been opened. All running transactions must complete before the 16 | method will be called on the underlying lmdb.Env. 17 | 18 | If an open transaction depends on a call to Env.SetMapSize then the Env will 19 | deadlock and block all future transactions. When using a Handler to 20 | automatically call Env.SetMapSize this implies the restriction that 21 | transactions must terminate independently of the creation/termination of other 22 | transactions to avoid deadlock. 23 | 24 | In the simplest example, a view transaction that attempts an update on the 25 | underlying Env will deadlock the environment if the map is full and a Handler 26 | attempts to resize the map so the update may be retried. 27 | 28 | env.View(func(txn *lmdb.Txn) (err error) { 29 | v, err := txn.Get(db, key) 30 | if err != nil { 31 | return err 32 | } 33 | err = env.Update(func(txn *lmdb.Txn) (err error) { // deadlock on lmdb.MapFull! 34 | txn.Put(dbi, key, append(v, b...)) 35 | }) 36 | return err 37 | } 38 | 39 | The update should instead be prepared inside the view and then executed 40 | following its termination. This removes the implicit dependence of the view on 41 | calls to Env.SetMapSize(). 42 | 43 | var v []byte 44 | env.View(func(txn *lmdb.Txn) (err error) { 45 | // RawRead isn't used because the value will be used outside the 46 | // transaction. 47 | v, err = txn.Get(db, key) 48 | if err != nil { 49 | return err 50 | } 51 | return nil 52 | } 53 | if err != nil { 54 | // ... 55 | } 56 | err = env.Update(func(txn *lmdb.Txn) (err error) { // no deadlock, even if env is resized! 57 | txn.Put(dbi, key, append(v, b...)) 58 | }) 59 | 60 | The developers of LMDB officially recommend against applications changing the 61 | memory map size for an open database. It requires careful synchronization by 62 | all processes accessing the database file. And, a large memory map will not 63 | affect disk usage on operating systems that support sparse files (e.g. Linux, 64 | not OS X). 65 | 66 | See mdb_env_set_mapsize. 67 | 68 | MapFull 69 | 70 | The MapFullHandler function configures an Env to automatically call increase 71 | the map size with Env.SetMapSize and retry transactions when a lmdb.MapFull 72 | error prevents an update from being committed. 73 | 74 | Because updates may need to execute multiple times in the presence of 75 | lmdb.MapFull it is important to make sure their TxnOp functions are idempotent 76 | and do not cause unwanted additive change to the program state. 77 | 78 | See mdb_txn_commit and MDB_MAP_FULL. 79 | 80 | MapResized 81 | 82 | When multiple processes access and resize an environment it is not uncommon to 83 | encounter a MapResized error which prevents the TxnOp from being executed and 84 | requires a synchronized call to Env.SetMapSize before continuing normal 85 | operation. 86 | 87 | The MapResizedHandler function configures an Env to automatically adopt a new 88 | map size when a lmdb.MapResized error is encountered and retry execution of the 89 | TxnOp. 90 | 91 | See mdb_txn_begin and MDB_MAP_RESIZED. 92 | 93 | NoLock 94 | 95 | When the lmdb.NoLock flag is set on an environment Env handles all transaction 96 | synchronization using Go structures and is an experimental feature. It is 97 | unclear what benefits this provides. 98 | 99 | Usage of lmdb.NoLock requires that update transactions acquire an exclusive 100 | lock on the environment. In such cases it is required that view transactions 101 | execute independently of update transactions, a requirement more strict than 102 | that from handling MapFull. 103 | 104 | See mdb_env_open and MDB_NOLOCK. 105 | */ 106 | package lmdbsync 107 | 108 | import ( 109 | "fmt" 110 | "os" 111 | "sync" 112 | "time" 113 | 114 | "golang.org/x/net/context" 115 | 116 | "github.com/bmatsuo/lmdb-go/lmdb" 117 | ) 118 | 119 | // Env wraps an *lmdb.Env, receiving all the same methods and proxying some to 120 | // provide transaction management. Transactions run by an Env handle 121 | // lmdb.MapResized error transparently through additional synchronization. 122 | // Additionally, Env is safe to use on environments setting the lmdb.NoLock 123 | // flag. When in NoLock mode write transactions block all read transactions 124 | // from running (in addition to blocking other write transactions like a normal 125 | // lmdb.Env would). 126 | // 127 | // Env proxies several methods to provide synchronization required for safe 128 | // operation in some scenarios. It is important not byprass proxies and call 129 | // the methods directly on the underlying lmdb.Env or synchronization may be 130 | // interfered with. Calling proxied methods directly on the lmdb.Env may 131 | // result in poor transaction performance or unspecified behavior in from the C 132 | // library. 133 | type Env struct { 134 | *lmdb.Env 135 | Handlers HandlerChain 136 | ctx context.Context 137 | noLock bool 138 | txnlock sync.RWMutex 139 | } 140 | 141 | // NewEnv returns an newly allocated Env that wraps env. If env is nil then 142 | // lmdb.NewEnv() will be called to allocate an lmdb.Env. 143 | func NewEnv(env *lmdb.Env, h ...Handler) (*Env, error) { 144 | var err error 145 | if env == nil { 146 | env, err = lmdb.NewEnv() 147 | if err != nil { 148 | return nil, err 149 | } 150 | } 151 | 152 | flags, err := env.Flags() 153 | if err != nil { 154 | return nil, err 155 | } 156 | noLock := flags&lmdb.NoLock != 0 157 | 158 | chain := append(HandlerChain(nil), h...) 159 | 160 | _env := &Env{ 161 | Env: env, 162 | Handlers: chain, 163 | noLock: noLock, 164 | ctx: context.Background(), 165 | } 166 | return _env, nil 167 | } 168 | 169 | // Open is a proxy for r.Env.Open() that detects the lmdb.NoLock flag to 170 | // properly manage transaction synchronization. 171 | func (r *Env) Open(path string, flags uint, mode os.FileMode) error { 172 | err := r.Env.Open(path, flags, mode) 173 | if err != nil { 174 | // no update to flags occurred 175 | return err 176 | } 177 | 178 | if flags&lmdb.NoLock != 0 { 179 | r.noLock = true 180 | } 181 | 182 | return nil 183 | } 184 | 185 | // SetMapSize is a proxy for r.Env.SetMapSize() that blocks while concurrent 186 | // transactions are in progress. 187 | func (r *Env) SetMapSize(size int64) error { 188 | return r.setMapSize(size, 0) 189 | } 190 | 191 | func (r *Env) setMapSize(size int64, delay time.Duration) error { 192 | r.txnlock.Lock() 193 | if delay > 0 { 194 | // wait before adopting a map size set from another process. hold on to 195 | // the transaction lock so that other transactions don't attempt to 196 | // begin while waiting. 197 | time.Sleep(delay) 198 | } 199 | err := r.Env.SetMapSize(size) 200 | r.txnlock.Unlock() 201 | return err 202 | } 203 | 204 | // BeginTxn overrides the r.Env.BeginTxn and always returns an error. An 205 | // unmanaged transaction. 206 | func (r *Env) BeginTxn(parent *lmdb.Txn, flags uint) (*lmdb.Txn, error) { 207 | return nil, fmt.Errorf("lmdbsync: unmanaged transactions are not supported") 208 | } 209 | 210 | // RunTxn is a proxy for r.Env.RunTxn(). 211 | // 212 | // If lmdb.NoLock is set on r.Env then RunTxn will block while other updates 213 | // are in progress, regardless of flags. 214 | func (r *Env) RunTxn(flags uint, op lmdb.TxnOp) (err error) { 215 | readonly := flags&lmdb.Readonly != 0 216 | return r.runHandler(readonly, func() error { return r.Env.RunTxn(flags, op) }, r.Handlers) 217 | } 218 | 219 | // View is a proxy for r.Env.View(). 220 | // 221 | // If lmdb.NoLock is set on r.Env then View will block until any running update 222 | // completes. 223 | func (r *Env) View(op lmdb.TxnOp) error { 224 | return r.runHandler(true, func() error { return r.Env.View(op) }, r.Handlers) 225 | } 226 | 227 | // Update is a proxy for r.Env.Update(). 228 | // 229 | // If lmdb.NoLock is set on r.Env then Update blocks until all other 230 | // transactions have terminated and blocks all other transactions from running 231 | // while in progress (including readonly transactions). 232 | func (r *Env) Update(op lmdb.TxnOp) error { 233 | return r.runHandler(false, func() error { return r.Env.Update(op) }, r.Handlers) 234 | } 235 | 236 | // UpdateLocked is a proxy for r.Env.UpdateLocked(). 237 | // 238 | // If lmdb.NoLock is set on r.Env then UpdateLocked blocks until all other 239 | // transactions have terminated and blocks all other transactions from running 240 | // while in progress (including readonly transactions). 241 | func (r *Env) UpdateLocked(op lmdb.TxnOp) error { 242 | return r.runHandler(false, func() error { return r.Env.UpdateLocked(op) }, r.Handlers) 243 | } 244 | 245 | // WithHandler returns a TxnRunner than handles transaction errors r.Handlers 246 | // chained with h. 247 | func (r *Env) WithHandler(h Handler) TxnRunner { 248 | return &handlerRunner{ 249 | env: r, 250 | h: r.Handlers.Append(h), 251 | } 252 | } 253 | 254 | func (r *Env) runHandler(readonly bool, fn func() error, h Handler) error { 255 | ctx := r.ctx 256 | for { 257 | err := r.run(readonly, fn) 258 | ctx, err = h.HandleTxnErr(ctx, r, err) 259 | if err != ErrTxnRetry { 260 | return err 261 | } 262 | } 263 | } 264 | func (r *Env) run(readonly bool, fn func() error) error { 265 | var err error 266 | if r.noLock && !readonly { 267 | r.txnlock.Lock() 268 | err = fn() 269 | r.txnlock.Unlock() 270 | } else { 271 | r.txnlock.RLock() 272 | err = fn() 273 | r.txnlock.RUnlock() 274 | } 275 | return err 276 | } 277 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/exp/lmdbsync/lmdbsync_test.go: -------------------------------------------------------------------------------- 1 | package lmdbsync 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "runtime" 7 | "syscall" 8 | "testing" 9 | "time" 10 | 11 | "github.com/bmatsuo/lmdb-go/internal/lmdbtest" 12 | "github.com/bmatsuo/lmdb-go/lmdb" 13 | ) 14 | 15 | var optNoLock = &lmdbtest.EnvOptions{Flags: lmdb.NoLock} 16 | 17 | func newEnv(opt *lmdbtest.EnvOptions) (*Env, error) { 18 | env, err := lmdbtest.NewEnv(opt) 19 | if err != nil { 20 | return nil, err 21 | } 22 | return NewEnv(env) 23 | } 24 | 25 | func TestNewEnv(t *testing.T) { 26 | dir, err := ioutil.TempDir("", "lmdbsync-test-") 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | defer os.RemoveAll(dir) 31 | 32 | env, err := NewEnv(nil) 33 | if err != nil { 34 | t.Error(err) 35 | return 36 | } 37 | defer env.Close() 38 | err = env.Open(dir, 0, 0644) 39 | if err != nil { 40 | t.Error(err) 41 | return 42 | } 43 | 44 | if env.noLock { 45 | t.Errorf("flag lmdb.NoLock detected incorrectly") 46 | } 47 | 48 | info, err := env.Info() 49 | if err != nil { 50 | t.Error(err) 51 | } 52 | if info.MapSize <= 0 { 53 | t.Errorf("bad mapsize: %v", info.MapSize) 54 | } 55 | } 56 | 57 | func TestEnv_Open(t *testing.T) { 58 | dir, err := ioutil.TempDir("", "lmdbsync-test-") 59 | if err != nil { 60 | t.Fatal(err) 61 | } 62 | defer os.RemoveAll(dir) 63 | 64 | env, err := NewEnv(nil) 65 | if err != nil { 66 | t.Error(err) 67 | return 68 | } 69 | defer env.Close() 70 | err = env.Open(dir, 0, 0644) 71 | if err != nil { 72 | t.Error(err) 73 | return 74 | } 75 | 76 | if env.noLock { 77 | t.Error("flag lmdb.NoLock detected incorrectly") 78 | } 79 | 80 | // calling Open on an open environment will fail. env.noLock should not be 81 | // set on a failing call to Open. 82 | err = env.Open(dir, lmdb.NoLock, 0644) 83 | if !lmdb.IsErrnoSys(err, syscall.EINVAL) { 84 | t.Error(err) 85 | } 86 | 87 | if env.noLock { 88 | t.Error("flag lmdb.NoLock detected incorrectly") 89 | } 90 | } 91 | 92 | func TestNewEnv_noLock(t *testing.T) { 93 | dir, err := ioutil.TempDir("", "lmdbsync-test-") 94 | if err != nil { 95 | t.Fatal(err) 96 | } 97 | defer os.RemoveAll(dir) 98 | 99 | env, err := NewEnv(nil) 100 | if err != nil { 101 | t.Error(err) 102 | return 103 | } 104 | defer env.Close() 105 | err = env.Open(dir, lmdb.NoLock, 0644) 106 | if err != nil { 107 | t.Error(err) 108 | return 109 | } 110 | 111 | if !env.noLock { 112 | t.Errorf("flag lmdb.NoLock not detected correctly") 113 | } 114 | 115 | info, err := env.Info() 116 | if err != nil { 117 | t.Error(err) 118 | } 119 | if info.MapSize <= 0 { 120 | t.Errorf("bad mapsize: %v", info.MapSize) 121 | } 122 | } 123 | 124 | func TestNewEnv_noLock2(t *testing.T) { 125 | env, err := newEnv(optNoLock) 126 | if err != nil { 127 | t.Error(err) 128 | return 129 | } 130 | defer lmdbtest.Destroy(env.Env) 131 | 132 | if !env.noLock { 133 | t.Errorf("flag lmdb.NoLock not detected correctly") 134 | } 135 | 136 | info, err := env.Info() 137 | if err != nil { 138 | t.Error(err) 139 | } 140 | if info.MapSize <= 0 { 141 | t.Errorf("bad mapsize: %v", info.MapSize) 142 | } 143 | } 144 | 145 | func TestNewEnv_arg(t *testing.T) { 146 | dir, err := ioutil.TempDir("", "lmdbsync-test-") 147 | if err != nil { 148 | t.Fatal(err) 149 | } 150 | defer os.RemoveAll(dir) 151 | 152 | _env, err := lmdb.NewEnv() 153 | if err != nil { 154 | t.Error(err) 155 | return 156 | } 157 | env, err := NewEnv(_env) 158 | if err != nil { 159 | t.Error(err) 160 | return 161 | } 162 | defer env.Close() 163 | 164 | if env.noLock { 165 | t.Errorf("flag lmdb.NoLock detected incorrectly") 166 | } 167 | 168 | err = env.Open(dir, lmdb.NoLock, 0644) 169 | if err != nil { 170 | t.Error(err) 171 | return 172 | } 173 | 174 | if !env.noLock { 175 | t.Errorf("flag lmdb.NoLock not detected correctly") 176 | } 177 | 178 | info, err := env.Info() 179 | if err != nil { 180 | t.Error(err) 181 | } 182 | if info.MapSize <= 0 { 183 | t.Errorf("bad mapsize: %v", info.MapSize) 184 | } 185 | } 186 | 187 | func TestNewEnv_noLock_arg(t *testing.T) { 188 | dir, err := ioutil.TempDir("", "lmdbsync-test-") 189 | if err != nil { 190 | t.Fatal(err) 191 | } 192 | defer os.RemoveAll(dir) 193 | 194 | _env, err := lmdb.NewEnv() 195 | if err != nil { 196 | t.Error(err) 197 | return 198 | } 199 | err = _env.Open(dir, lmdb.NoLock, 0644) 200 | if err != nil { 201 | t.Error(err) 202 | return 203 | } 204 | 205 | env, err := NewEnv(_env) 206 | if err != nil { 207 | t.Error(err) 208 | return 209 | } 210 | defer env.Close() 211 | 212 | if !env.noLock { 213 | t.Errorf("flag lmdb.NoLock not detected correctly") 214 | } 215 | 216 | info, err := env.Info() 217 | if err != nil { 218 | t.Error(err) 219 | } 220 | if info.MapSize <= 0 { 221 | t.Errorf("bad mapsize: %v", info.MapSize) 222 | } 223 | } 224 | 225 | func TestEnv_SetMapSize(t *testing.T) { 226 | env, err := newEnv(nil) 227 | if err != nil { 228 | t.Fatal(err) 229 | } 230 | defer lmdbtest.Destroy(env.Env) 231 | 232 | txnopen := make(chan struct{}) 233 | errc := make(chan error, 1) 234 | go func() { 235 | // open a transaction, signal to the main routine than the transaction 236 | // is open, and wait for a short period. 237 | errc <- env.View(func(txn *lmdb.Txn) (err error) { 238 | txnopen <- struct{}{} 239 | time.Sleep(50 * time.Millisecond) 240 | return nil 241 | }) 242 | }() 243 | 244 | // once the transaction has been opened attempt to change the map size. 245 | // the call to SetMapSize will block until the transaction completes. 246 | <-txnopen 247 | err = env.SetMapSize(10 << 20) 248 | if err != nil { 249 | t.Error(err) 250 | } 251 | 252 | // finally check for any error in the transaction. 253 | err = <-errc 254 | if err != nil { 255 | t.Error(err) 256 | } 257 | 258 | } 259 | 260 | func TestEnv_BeginTxn(t *testing.T) { 261 | env, err := newEnv(nil) 262 | if err != nil { 263 | t.Fatal(err) 264 | } 265 | defer lmdbtest.Destroy(env.Env) 266 | 267 | txn, err := env.BeginTxn(nil, 0) 268 | if err == nil { 269 | t.Error("transaction was created") 270 | txn.Abort() 271 | } 272 | } 273 | 274 | func testView(t *testing.T, env TxnRunner) { 275 | err := env.View(func(txn *lmdb.Txn) (err error) { 276 | dbi, err := txn.OpenRoot(0) 277 | if err != nil { 278 | return err 279 | } 280 | err = txn.Put(dbi, []byte("k"), []byte("v"), 0) 281 | if err == nil { 282 | t.Error("put allowed inside view transaction") 283 | } 284 | return nil 285 | }) 286 | if err != nil { 287 | t.Error(err) 288 | } 289 | } 290 | 291 | func testUpdate(t *testing.T, env TxnRunner) { 292 | var dbi lmdb.DBI 293 | err := env.Update(func(txn *lmdb.Txn) (err error) { 294 | dbi, err = txn.OpenRoot(0) 295 | if err != nil { 296 | return err 297 | } 298 | return txn.Put(dbi, []byte("k"), []byte("v"), 0) 299 | }) 300 | if err != nil { 301 | t.Error(err) 302 | } 303 | 304 | err = env.View(func(txn *lmdb.Txn) (err error) { 305 | v, err := txn.Get(dbi, []byte("k")) 306 | if err != nil { 307 | return err 308 | } 309 | if string(v) != "v" { 310 | t.Errorf("unexpected value: %q (!= %q)", v, "v") 311 | } 312 | return nil 313 | }) 314 | if err != nil { 315 | t.Error(err) 316 | } 317 | } 318 | 319 | func testUpdateLocked(t *testing.T, env TxnRunner) { 320 | runtime.LockOSThread() 321 | defer runtime.UnlockOSThread() 322 | var dbi lmdb.DBI 323 | err := env.UpdateLocked(func(txn *lmdb.Txn) (err error) { 324 | dbi, err = txn.OpenRoot(0) 325 | if err != nil { 326 | return err 327 | } 328 | return txn.Put(dbi, []byte("k"), []byte("v"), 0) 329 | }) 330 | if err != nil { 331 | t.Error(err) 332 | } 333 | 334 | err = env.View(func(txn *lmdb.Txn) (err error) { 335 | v, err := txn.Get(dbi, []byte("k")) 336 | if err != nil { 337 | return err 338 | } 339 | if string(v) != "v" { 340 | t.Errorf("unexpected value: %q (!= %q)", v, "v") 341 | } 342 | return nil 343 | }) 344 | if err != nil { 345 | t.Error(err) 346 | } 347 | } 348 | 349 | func testRunTxn(t *testing.T, env TxnRunner) { 350 | var dbi lmdb.DBI 351 | err := env.RunTxn(0, func(txn *lmdb.Txn) (err error) { 352 | dbi, err = txn.OpenRoot(0) 353 | if err != nil { 354 | return err 355 | } 356 | return txn.Put(dbi, []byte("k"), []byte("v"), 0) 357 | }) 358 | if err != nil { 359 | t.Error(err) 360 | } 361 | 362 | err = env.RunTxn(lmdb.Readonly, func(txn *lmdb.Txn) (err error) { 363 | dbi, err := txn.OpenRoot(0) 364 | if err != nil { 365 | return err 366 | } 367 | err = txn.Put(dbi, []byte("k"), []byte("V"), 0) 368 | if err == nil { 369 | t.Error("put allowed inside view transaction") 370 | } 371 | return nil 372 | }) 373 | if err != nil { 374 | t.Error(err) 375 | } 376 | 377 | err = env.View(func(txn *lmdb.Txn) (err error) { 378 | v, err := txn.Get(dbi, []byte("k")) 379 | if err != nil { 380 | return err 381 | } 382 | if string(v) != "v" { 383 | t.Errorf("unexpected value: %q (!= %q)", v, "v") 384 | } 385 | return nil 386 | }) 387 | if err != nil { 388 | t.Error(err) 389 | } 390 | } 391 | 392 | func TestEnv_View(t *testing.T) { 393 | env, err := newEnv(nil) 394 | if err != nil { 395 | t.Fatal(err) 396 | } 397 | defer lmdbtest.Destroy(env.Env) 398 | 399 | testView(t, env) 400 | } 401 | 402 | func TestEnv_Update(t *testing.T) { 403 | env, err := newEnv(nil) 404 | if err != nil { 405 | t.Fatal(err) 406 | } 407 | defer lmdbtest.Destroy(env.Env) 408 | 409 | testUpdate(t, env) 410 | } 411 | 412 | func TestEnv_UpdateLocked(t *testing.T) { 413 | env, err := newEnv(nil) 414 | if err != nil { 415 | t.Fatal(err) 416 | } 417 | defer lmdbtest.Destroy(env.Env) 418 | 419 | testUpdateLocked(t, env) 420 | } 421 | 422 | func TestEnv_RunTxn(t *testing.T) { 423 | env, err := newEnv(nil) 424 | if err != nil { 425 | t.Fatal(err) 426 | } 427 | defer lmdbtest.Destroy(env.Env) 428 | 429 | testRunTxn(t, env) 430 | } 431 | 432 | func TestEnv_View_NoLock(t *testing.T) { 433 | env, err := newEnv(optNoLock) 434 | if err != nil { 435 | t.Fatal(err) 436 | } 437 | defer lmdbtest.Destroy(env.Env) 438 | 439 | testView(t, env) 440 | } 441 | 442 | func TestEnv_Update_NoLock(t *testing.T) { 443 | env, err := newEnv(optNoLock) 444 | if err != nil { 445 | t.Fatal(err) 446 | } 447 | defer lmdbtest.Destroy(env.Env) 448 | 449 | testUpdate(t, env) 450 | } 451 | 452 | func TestEnv_UpdateLocked_NoLock(t *testing.T) { 453 | env, err := newEnv(optNoLock) 454 | if err != nil { 455 | t.Fatal(err) 456 | } 457 | defer lmdbtest.Destroy(env.Env) 458 | 459 | testUpdateLocked(t, env) 460 | } 461 | 462 | func TestEnv_RunTxn_NoLock(t *testing.T) { 463 | env, err := newEnv(optNoLock) 464 | if err != nil { 465 | t.Fatal(err) 466 | } 467 | defer lmdbtest.Destroy(env.Env) 468 | 469 | testRunTxn(t, env) 470 | } 471 | 472 | func TestEnv_WithHandler_View(t *testing.T) { 473 | env, err := newEnv(nil) 474 | if err != nil { 475 | t.Fatal(err) 476 | } 477 | defer lmdbtest.Destroy(env.Env) 478 | 479 | handler := &testHandler{} 480 | runner := env.WithHandler(handler) 481 | 482 | testView(t, runner) 483 | 484 | if handler.env != env { 485 | t.Errorf("handler does not include original env") 486 | } 487 | if !handler.called { 488 | t.Errorf("handler was not called") 489 | } 490 | } 491 | 492 | func TestEnv_WithHandler_Update(t *testing.T) { 493 | env, err := newEnv(nil) 494 | if err != nil { 495 | t.Fatal(err) 496 | } 497 | defer lmdbtest.Destroy(env.Env) 498 | 499 | handler := &testHandler{} 500 | runner := env.WithHandler(handler) 501 | 502 | testUpdate(t, runner) 503 | 504 | if handler.env != env { 505 | t.Errorf("handler does not include original env") 506 | } 507 | if !handler.called { 508 | t.Errorf("handler was not called") 509 | } 510 | } 511 | 512 | func TestEnv_WithHandler_UpdateLocked(t *testing.T) { 513 | env, err := newEnv(nil) 514 | if err != nil { 515 | t.Fatal(err) 516 | } 517 | defer lmdbtest.Destroy(env.Env) 518 | 519 | handler := &testHandler{} 520 | runner := env.WithHandler(handler) 521 | 522 | testUpdateLocked(t, runner) 523 | 524 | if handler.env != env { 525 | t.Errorf("handler does not include original env") 526 | } 527 | if !handler.called { 528 | t.Errorf("handler was not called") 529 | } 530 | } 531 | 532 | func TestEnv_WithHandler_RunTxn(t *testing.T) { 533 | env, err := newEnv(nil) 534 | if err != nil { 535 | t.Fatal(err) 536 | } 537 | defer lmdbtest.Destroy(env.Env) 538 | 539 | handler := &testHandler{} 540 | runner := env.WithHandler(handler) 541 | 542 | testRunTxn(t, runner) 543 | 544 | if handler.env != env { 545 | t.Errorf("handler does not include original env") 546 | } 547 | if !handler.called { 548 | t.Errorf("handler was not called") 549 | } 550 | } 551 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/exp/lmdbsync/testresize/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Command testresize is a utility used by the lmdbsync tests to validate its 3 | multiprocessing capabilities. An external command like testresize command is 4 | required because a process is not allowed to map the same process twice. 5 | 6 | Testresize writes batches of updates into a databaes, transparently handling 7 | any lmdb.MapResized or lmdb.MapFull errors that occur. To ensure that resizing 8 | behavior is observed testresize waits for input before updating the environment 9 | and writes a line to stdout after the update is committed. If testresize 10 | process observes zero of either error it will exit with a non-zero exit code. 11 | 12 | Two testresize processes can communicate to each other using two unix pipes, if 13 | the output of each pipe is connected to the input of the other. Writing a 14 | single line to one of the pipes will cause updates to ping-pong back and forth 15 | between processes. 16 | */ 17 | package main 18 | 19 | import ( 20 | "bufio" 21 | "crypto/rand" 22 | "flag" 23 | "fmt" 24 | "io" 25 | "log" 26 | "os" 27 | "time" 28 | 29 | "golang.org/x/net/context" 30 | 31 | "github.com/bmatsuo/lmdb-go/exp/lmdbsync" 32 | "github.com/bmatsuo/lmdb-go/lmdb" 33 | ) 34 | 35 | func main() { 36 | numitems := flag.Int64("n", 5<<10, "the number of items to write") 37 | chunksize := flag.Int64("c", 100, "the number of items to write per txn") 38 | flag.Parse() 39 | 40 | failed := false 41 | defer func() { 42 | if failed { 43 | os.Exit(1) 44 | } 45 | }() 46 | fail := func(err error) { 47 | failed = true 48 | log.Print(err) 49 | } 50 | 51 | err := WriteRandomItems("db", *numitems, *chunksize) 52 | if err != nil { 53 | fail(err) 54 | } else { 55 | log.Printf("success") 56 | } 57 | } 58 | 59 | // WriteRandomItems writes numitem items with chunksize sized values full of 60 | // random data. 61 | func WriteRandomItems(path string, numitem, chunksize int64) (err error) { 62 | env, err := OpenEnv(path) 63 | if err != nil { 64 | return err 65 | } 66 | defer env.Close() 67 | 68 | numResize := 0 69 | numResized := 0 70 | defer func() { 71 | log.Printf("%d resizes", numResize) 72 | log.Printf("%d size adoptions", numResized) 73 | if err == nil { 74 | if numResize == 0 { 75 | err = fmt.Errorf("process did not resize the memory map") 76 | } else if numResized == 0 { 77 | err = fmt.Errorf("process did not adopt a new map size") 78 | } 79 | } 80 | }() 81 | mapResizedLogger := func(ctx context.Context, env *lmdbsync.Env, err error) (context.Context, error) { 82 | if lmdb.IsMapResized(err) { 83 | log.Printf("map resized") 84 | numResized++ 85 | } 86 | return ctx, err 87 | } 88 | mapFullLogger := func(ctx context.Context, env *lmdbsync.Env, err error) (context.Context, error) { 89 | if lmdb.IsMapFull(err) { 90 | log.Printf("resize required") 91 | numResize++ 92 | } 93 | return ctx, err 94 | } 95 | env.Handlers = env.Handlers.Append( 96 | handlerFunc(mapResizedLogger), 97 | lmdbsync.MapResizedHandler(2, func(int) time.Duration { return 100 * time.Microsecond }), 98 | handlerFunc(mapFullLogger), 99 | lmdbsync.MapFullHandler(func(size int64) (int64, bool) { 100 | newsize := size + 128<<10 // linear scale is bad -- but useful to test 101 | log.Printf("oldsize=%d newsize=%d", size, newsize) 102 | return newsize, true 103 | }), 104 | ) 105 | 106 | pid := os.Getpid() 107 | 108 | scanner := bufio.NewScanner(os.Stdin) 109 | for i := int64(0); i < numitem; { 110 | if !scanner.Scan() { 111 | return scanner.Err() 112 | } 113 | 114 | start := i 115 | chunkmax := i + chunksize 116 | if chunkmax > numitem { 117 | chunkmax = numitem 118 | } 119 | v := make([]byte, 512) 120 | _, err := io.ReadFull(rand.Reader, v) 121 | if err != nil { 122 | return err 123 | } 124 | err = env.Update(func(txn *lmdb.Txn) (err error) { 125 | dbi, err := txn.OpenRoot(0) 126 | if err != nil { 127 | return err 128 | } 129 | 130 | for i = start; i < chunkmax; i++ { 131 | k := fmt.Sprintf("%06d-%016x", pid, i) 132 | err = txn.Put(dbi, []byte(k), v, 0) 133 | if err != nil { 134 | return err 135 | } 136 | } 137 | 138 | return nil 139 | }) 140 | if err != nil { 141 | return err 142 | } 143 | fmt.Println("ok") 144 | } 145 | 146 | return nil 147 | } 148 | 149 | // OpenEnv is a helper for opening an lmdbsync.Env. 150 | func OpenEnv(path string) (*lmdbsync.Env, error) { 151 | env, err := lmdbsync.NewEnv(nil) 152 | if err != nil { 153 | return nil, err 154 | } 155 | err = env.Open(path, 0, 0644) 156 | if err != nil { 157 | env.Close() 158 | return nil, err 159 | } 160 | return env, nil 161 | } 162 | 163 | type handlerFunc func(ctx context.Context, env *lmdbsync.Env, err error) (context.Context, error) 164 | 165 | func (fn handlerFunc) HandleTxnErr(ctx context.Context, env *lmdbsync.Env, err error) (context.Context, error) { 166 | return fn(ctx, env, err) 167 | } 168 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/exp/lmdbsync/testresize_test.go: -------------------------------------------------------------------------------- 1 | package lmdbsync 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | "testing" 12 | "time" 13 | 14 | "golang.org/x/net/context" 15 | 16 | "github.com/bmatsuo/lmdb-go/lmdb" 17 | ) 18 | 19 | func TestResize(t *testing.T) { 20 | tempdir, err := ioutil.TempDir("", "lmdbsync_testresize") 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | defer os.RemoveAll(tempdir) 25 | 26 | dbpath := filepath.Join(tempdir, "db") 27 | err = os.Mkdir(dbpath, 0755) 28 | if err != nil { 29 | t.Error(err) 30 | return 31 | } 32 | 33 | env, err := NewEnv(nil) 34 | if err != nil { 35 | t.Error(err) 36 | return 37 | } 38 | defer env.Close() 39 | err = env.Open(dbpath, 0, 0644) 40 | if err != nil { 41 | t.Error(err) 42 | return 43 | } 44 | 45 | var root lmdb.DBI 46 | env.Update(func(txn *lmdb.Txn) (err error) { 47 | root, err = txn.OpenRoot(0) 48 | if err != nil { 49 | return err 50 | } 51 | return txn.Put(root, []byte("_start"), []byte(time.Now().String()), 0) 52 | }) 53 | 54 | before, err := env.Info() 55 | if err != nil { 56 | t.Error(err) 57 | return 58 | } 59 | 60 | bin := filepath.Join(tempdir, "testresize") 61 | build := exec.Command("go", "build", "-o", bin, "./testresize") 62 | build.Stderr = os.Stderr 63 | err = build.Run() 64 | if err != nil { 65 | t.Error(err) 66 | return 67 | } 68 | 69 | r1, w1, err := os.Pipe() 70 | if err != nil { 71 | t.Error(err) 72 | return 73 | } 74 | r2, w2, err := os.Pipe() 75 | if err != nil { 76 | t.Error(err) 77 | return 78 | } 79 | closePipes := func() { 80 | w1.Close() 81 | w2.Close() 82 | r1.Close() 83 | r2.Close() 84 | } 85 | 86 | cmd1 := exec.Command(bin) 87 | cmd1.Dir = tempdir 88 | cmd2 := exec.Command(bin) 89 | cmd2.Dir = tempdir 90 | 91 | cmd1.Stdin = r1 92 | cmd2.Stdout = w1 93 | 94 | cmd2.Stdin = r2 95 | cmd1.Stdout = w2 96 | 97 | stderr1, err := cmd1.StderrPipe() 98 | if err != nil { 99 | t.Error(err) 100 | closePipes() 101 | return 102 | } 103 | stderr2, err := cmd2.StderrPipe() 104 | if err != nil { 105 | t.Error(err) 106 | closePipes() 107 | return 108 | } 109 | stderr := io.MultiReader(stderr1, stderr2) 110 | 111 | err = cmd1.Start() 112 | if err != nil { 113 | t.Errorf("start 1: %v", err) 114 | closePipes() 115 | return 116 | } 117 | err = cmd2.Start() 118 | if err != nil { 119 | t.Errorf("start 2: %v", err) 120 | closePipes() 121 | return 122 | } 123 | go fmt.Fprintln(w1, "start") 124 | 125 | scanner := bufio.NewScanner(stderr) 126 | for scanner.Scan() { 127 | t.Log(scanner.Text()) 128 | } 129 | closePipes() 130 | 131 | err1 := cmd1.Wait() 132 | if err1 != nil { 133 | t.Errorf("err 1: %v", err1) 134 | } 135 | err2 := cmd2.Wait() 136 | if err2 != nil { 137 | t.Errorf("err 2: %v", err2) 138 | } 139 | 140 | trace := &resizeTracer{} 141 | runner := env.WithHandler(HandlerChain{ 142 | trace, 143 | MapResizedHandler(2, func(retry int) time.Duration { 144 | if retry > 0 { 145 | t.Errorf("failed to reopen at %d times", retry) 146 | } 147 | return time.Millisecond 148 | }), 149 | }) 150 | err = runner.Update(func(txn *lmdb.Txn) (err error) { 151 | return txn.Put(root, []byte("_finish"), []byte(time.Now().String()), 0) 152 | }) 153 | if err != nil { 154 | t.Error(err) 155 | } 156 | 157 | if trace.resized == 0 { 158 | t.Errorf("no resize detected") 159 | } 160 | 161 | after, err := env.Info() 162 | if err != nil { 163 | t.Error(err) 164 | return 165 | } 166 | if after.MapSize <= before.MapSize { 167 | t.Errorf("mapsize: %d (<= %d)", after.MapSize, before.MapSize) 168 | } 169 | } 170 | 171 | type resizeTracer struct { 172 | resized int 173 | } 174 | 175 | func (t *resizeTracer) HandleTxnErr(c context.Context, env *Env, err error) (context.Context, error) { 176 | if lmdb.IsMapResized(err) { 177 | t.resized++ 178 | } 179 | return c, err 180 | } 181 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/internal/lmdbcmd/cmutil.go: -------------------------------------------------------------------------------- 1 | package lmdbcmd 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io" 7 | "os" 8 | 9 | "github.com/bmatsuo/lmdb-go/lmdb" 10 | ) 11 | 12 | var flagPrintVersion bool 13 | var flagOpenNoSubDir bool 14 | 15 | func init() { 16 | flag.BoolVar(&flagPrintVersion, "V", false, "Write the library version number to the standard output, and exit.") 17 | flag.BoolVar(&flagOpenNoSubDir, "n", false, "Open LDMB environment(s) which do not use subdirectories.") 18 | } 19 | 20 | func printVersion(w io.Writer) { 21 | fmt.Fprintln(w, lmdb.VersionString()) 22 | } 23 | 24 | // PrintVersion writes the LMDB API version in a human readable format to 25 | // os.Stdout. 26 | func PrintVersion() { 27 | if flagPrintVersion { 28 | printVersion(os.Stdout) 29 | os.Exit(0) 30 | } 31 | } 32 | 33 | // OpenFlag returns the bitwise OR'd set of flags specified by options defined 34 | // in the package. The returned value may be OR'd with additional flags if 35 | // needed. 36 | func OpenFlag() uint { 37 | var flag uint 38 | if flagOpenNoSubDir { 39 | flag |= lmdb.NoSubdir 40 | } 41 | return flag 42 | } 43 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/internal/lmdbtest/lmdbtest.go: -------------------------------------------------------------------------------- 1 | package lmdbtest 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | 7 | "github.com/bmatsuo/lmdb-go/lmdb" 8 | ) 9 | 10 | // ItemList is a list of database items. 11 | type ItemList interface { 12 | Len() int 13 | Item(i int) Item 14 | } 15 | 16 | // EnvOptions specifies an environment configuration for a test involving an 17 | // LMDB database. 18 | type EnvOptions struct { 19 | MaxReaders int 20 | MaxDBs int 21 | MapSize int64 22 | Flags uint 23 | } 24 | 25 | // NewEnv returns a test environment with the given options at a temporary 26 | // path. 27 | func NewEnv(opt *EnvOptions) (env *lmdb.Env, err error) { 28 | var dir string 29 | defer func() { 30 | if err != nil { 31 | if env != nil { 32 | env.Close() 33 | } 34 | if dir != "" { 35 | os.RemoveAll(dir) 36 | } 37 | } 38 | }() 39 | 40 | dir, err = ioutil.TempDir("", "lmdbtest-env-") 41 | if err != nil { 42 | return nil, err 43 | } 44 | env, err = lmdb.NewEnv() 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | var maxreaders int 50 | var maxdbs int 51 | var mapsize int64 52 | var flags uint 53 | if opt != nil { 54 | maxreaders = opt.MaxReaders 55 | maxdbs = opt.MaxDBs 56 | mapsize = opt.MapSize 57 | flags = opt.Flags 58 | } 59 | 60 | if maxreaders != 0 { 61 | err = env.SetMaxReaders(maxreaders) 62 | if err != nil { 63 | return nil, err 64 | } 65 | } 66 | if maxdbs != 0 { 67 | err = env.SetMaxDBs(maxdbs) 68 | if err != nil { 69 | return nil, err 70 | } 71 | } 72 | if mapsize != 0 { 73 | err = env.SetMapSize(mapsize) 74 | if err != nil { 75 | return nil, err 76 | } 77 | } 78 | err = env.Open(dir, flags, 0644) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | return env, nil 84 | } 85 | 86 | // Destroy closes env and removes its directory from the file system. 87 | func Destroy(env *lmdb.Env) { 88 | if env == nil { 89 | return 90 | } 91 | path, _ := env.Path() 92 | env.Close() 93 | if path != "" { 94 | os.RemoveAll(path) 95 | } 96 | } 97 | 98 | // OpenDBI is a helper that opens a transaction, opens the named database, 99 | // commits the transaction, and returns the open DBI handle to the caller. 100 | func OpenDBI(env *lmdb.Env, name string, flags uint) (lmdb.DBI, error) { 101 | var dbi lmdb.DBI 102 | err := env.Update(func(txn *lmdb.Txn) (err error) { 103 | dbi, err = txn.OpenDBI(name, flags) 104 | return err 105 | }) 106 | return dbi, err 107 | } 108 | 109 | // OpenRoot is a helper that opens a transaction, opens the root database, 110 | // commits the transaction, and returns the open DBI handle to the caller. 111 | func OpenRoot(env *lmdb.Env, flags uint) (lmdb.DBI, error) { 112 | var dbi lmdb.DBI 113 | err := env.Update(func(txn *lmdb.Txn) (err error) { 114 | dbi, err = txn.OpenRoot(flags) 115 | return err 116 | }) 117 | return dbi, err 118 | } 119 | 120 | // Put writes items to the handle dbi in env. 121 | func Put(env *lmdb.Env, dbi lmdb.DBI, items ItemList) error { 122 | return env.Update(func(txn *lmdb.Txn) (err error) { 123 | for i, n := 0, items.Len(); i < n; i++ { 124 | item := items.Item(i) 125 | err = txn.Put(dbi, item.Key(), item.Val(), 0) 126 | if err != nil { 127 | return err 128 | } 129 | } 130 | return nil 131 | }) 132 | } 133 | 134 | // Item is an interface for database items. 135 | type Item interface { 136 | Key() []byte 137 | Val() []byte 138 | } 139 | 140 | // SimpleItemList is an ItemList 141 | type SimpleItemList []*SimpleItem 142 | 143 | // Len implements the ItemList interface. 144 | func (items SimpleItemList) Len() int { return len(items) } 145 | 146 | // Item implements the ItemList interface. 147 | func (items SimpleItemList) Item(i int) Item { return items[i] } 148 | 149 | // SimpleItem is a simple representation of a database item. 150 | type SimpleItem struct { 151 | K string 152 | V string 153 | } 154 | 155 | // Key implements Item interface. 156 | func (i *SimpleItem) Key() []byte { return []byte(i.K) } 157 | 158 | // Val implements the Item interface. 159 | func (i *SimpleItem) Val() []byte { return []byte(i.V) } 160 | 161 | // Len implements the ItemList interface 162 | func (i *SimpleItem) Len() int { return 1 } 163 | 164 | // Item implements the ItemList interface 165 | func (i *SimpleItem) Item(j int) Item { 166 | if j != 0 { 167 | panic("index out of range") 168 | } 169 | return i 170 | } 171 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/bench_test.go: -------------------------------------------------------------------------------- 1 | package lmdb 2 | 3 | import ( 4 | crand "crypto/rand" 5 | "math/rand" 6 | "os" 7 | "sync/atomic" 8 | "testing" 9 | ) 10 | 11 | func BenchmarkEnv_ReaderList(b *testing.B) { 12 | env := setup(b) 13 | defer clean(env, b) 14 | 15 | var txns []*Txn 16 | defer func() { 17 | for i, txn := range txns { 18 | if txn != nil { 19 | txn.Abort() 20 | txns[i] = nil 21 | } 22 | } 23 | }() 24 | 25 | const numreaders = 100 26 | for i := 0; i < numreaders; i++ { 27 | txn, err := env.BeginTxn(nil, Readonly) 28 | if err != nil { 29 | b.Error(err) 30 | return 31 | } 32 | txns = append(txns, txn) 33 | } 34 | 35 | b.ResetTimer() 36 | for i := 0; i < b.N; i++ { 37 | list := new(readerList) 38 | err := env.ReaderList(list.Next) 39 | if err != nil { 40 | b.Error(err) 41 | return 42 | } 43 | if list.Len() != numreaders+1 { 44 | b.Errorf("reader list length: %v", list.Len()) 45 | } 46 | } 47 | } 48 | 49 | type readerList struct { 50 | ln []string 51 | } 52 | 53 | func (r *readerList) Len() int { 54 | return len(r.ln) 55 | } 56 | 57 | func (r *readerList) Next(ln string) error { 58 | r.ln = append(r.ln, ln) 59 | return nil 60 | } 61 | 62 | // repeatedly put (overwrite) keys. 63 | func BenchmarkTxn_Put(b *testing.B) { 64 | initRandSource(b) 65 | env := setup(b) 66 | defer clean(env, b) 67 | 68 | dbi := openBenchDBI(b, env) 69 | 70 | rc := newRandSourceCursor() 71 | ps, err := populateBenchmarkDB(env, dbi, &rc) 72 | if err != nil { 73 | b.Errorf("populate db: %v", err) 74 | return 75 | } 76 | 77 | err = env.Update(func(txn *Txn) (err error) { 78 | b.ResetTimer() 79 | defer b.StopTimer() 80 | for i := 0; i < b.N; i++ { 81 | k := ps[rand.Intn(len(ps)/2)*2] 82 | v := makeBenchDBVal(&rc) 83 | err := txn.Put(dbi, k, v, 0) 84 | if err != nil { 85 | return err 86 | } 87 | } 88 | return nil 89 | }) 90 | if err != nil { 91 | b.Error(err) 92 | return 93 | } 94 | } 95 | 96 | // repeatedly put (overwrite) keys using the PutReserve method. 97 | func BenchmarkTxn_PutReserve(b *testing.B) { 98 | initRandSource(b) 99 | env := setup(b) 100 | defer clean(env, b) 101 | 102 | dbi := openBenchDBI(b, env) 103 | 104 | rc := newRandSourceCursor() 105 | ps, err := populateBenchmarkDB(env, dbi, &rc) 106 | if err != nil { 107 | b.Errorf("populate db: %v", err) 108 | return 109 | } 110 | 111 | err = env.Update(func(txn *Txn) (err error) { 112 | b.ResetTimer() 113 | defer b.StopTimer() 114 | for i := 0; i < b.N; i++ { 115 | k := ps[rand.Intn(len(ps)/2)*2] 116 | v := makeBenchDBVal(&rc) 117 | buf, err := txn.PutReserve(dbi, k, len(v), 0) 118 | if err != nil { 119 | return err 120 | } 121 | copy(buf, v) 122 | } 123 | return nil 124 | }) 125 | if err != nil { 126 | b.Error(err) 127 | return 128 | } 129 | } 130 | 131 | // repeatedly put (overwrite) keys using the PutReserve method on an 132 | // environment with WriteMap. 133 | func BenchmarkTxn_PutReserve_writemap(b *testing.B) { 134 | initRandSource(b) 135 | env := setupFlags(b, WriteMap) 136 | defer clean(env, b) 137 | 138 | dbi := openBenchDBI(b, env) 139 | 140 | rc := newRandSourceCursor() 141 | ps, err := populateBenchmarkDB(env, dbi, &rc) 142 | if err != nil { 143 | b.Errorf("populate db: %v", err) 144 | return 145 | } 146 | 147 | err = env.Update(func(txn *Txn) (err error) { 148 | b.ResetTimer() 149 | defer b.StopTimer() 150 | for i := 0; i < b.N; i++ { 151 | k := ps[rand.Intn(len(ps)/2)*2] 152 | v := makeBenchDBVal(&rc) 153 | buf, err := txn.PutReserve(dbi, k, len(v), 0) 154 | if err != nil { 155 | return err 156 | } 157 | copy(buf, v) 158 | } 159 | return nil 160 | }) 161 | if err != nil { 162 | b.Error(err) 163 | return 164 | } 165 | } 166 | 167 | // repeatedly put (overwrite) keys. 168 | func BenchmarkTxn_Put_writemap(b *testing.B) { 169 | initRandSource(b) 170 | env := setupFlags(b, WriteMap) 171 | defer clean(env, b) 172 | 173 | dbi := openBenchDBI(b, env) 174 | 175 | var ps [][]byte 176 | 177 | rc := newRandSourceCursor() 178 | ps, err := populateBenchmarkDB(env, dbi, &rc) 179 | if err != nil { 180 | b.Errorf("populate db: %v", err) 181 | return 182 | } 183 | 184 | err = env.Update(func(txn *Txn) (err error) { 185 | b.ResetTimer() 186 | defer b.StopTimer() 187 | for i := 0; i < b.N; i++ { 188 | k := ps[rand.Intn(len(ps)/2)*2] 189 | v := makeBenchDBVal(&rc) 190 | err := txn.Put(dbi, k, v, 0) 191 | if err != nil { 192 | return err 193 | } 194 | } 195 | 196 | return nil 197 | }) 198 | if err != nil { 199 | b.Error(err) 200 | } 201 | } 202 | 203 | // repeatedly get random keys. 204 | func BenchmarkTxn_Get_ro(b *testing.B) { 205 | initRandSource(b) 206 | env := setup(b) 207 | defer clean(env, b) 208 | 209 | dbi := openBenchDBI(b, env) 210 | 211 | rc := newRandSourceCursor() 212 | ps, err := populateBenchmarkDB(env, dbi, &rc) 213 | if err != nil { 214 | b.Errorf("populate db: %v", err) 215 | return 216 | } 217 | 218 | err = env.View(func(txn *Txn) (err error) { 219 | b.ResetTimer() 220 | defer b.StopTimer() 221 | for i := 0; i < b.N; i++ { 222 | _, err := txn.Get(dbi, ps[rand.Intn(len(ps))]) 223 | if IsNotFound(err) { 224 | continue 225 | } 226 | if err != nil { 227 | b.Fatalf("error getting data: %v", err) 228 | } 229 | } 230 | 231 | return nil 232 | }) 233 | if err != nil { 234 | b.Error(err) 235 | } 236 | } 237 | 238 | // like BenchmarkTxnGetReadonly but txn.RawRead is set to true. 239 | func BenchmarkTxn_Get_raw_ro(b *testing.B) { 240 | initRandSource(b) 241 | env := setup(b) 242 | defer clean(env, b) 243 | 244 | dbi := openBenchDBI(b, env) 245 | 246 | rc := newRandSourceCursor() 247 | ps, err := populateBenchmarkDB(env, dbi, &rc) 248 | if err != nil { 249 | b.Errorf("populate db: %v", err) 250 | return 251 | } 252 | 253 | err = env.View(func(txn *Txn) (err error) { 254 | txn.RawRead = true 255 | b.ResetTimer() 256 | defer b.StopTimer() 257 | for i := 0; i < b.N; i++ { 258 | _, err := txn.Get(dbi, ps[rand.Intn(len(ps))]) 259 | if IsNotFound(err) { 260 | continue 261 | } 262 | if err != nil { 263 | b.Fatalf("error getting data: %v", err) 264 | } 265 | } 266 | return nil 267 | }) 268 | if err != nil { 269 | b.Error(err) 270 | return 271 | } 272 | } 273 | 274 | // repeatedly scan all the values in a database. 275 | func BenchmarkScan_ro(b *testing.B) { 276 | initRandSource(b) 277 | env := setup(b) 278 | defer clean(env, b) 279 | 280 | dbi := openBenchDBI(b, env) 281 | 282 | rc := newRandSourceCursor() 283 | _, err := populateBenchmarkDB(env, dbi, &rc) 284 | if err != nil { 285 | b.Errorf("populate db: %v", err) 286 | return 287 | } 288 | 289 | err = env.View(func(txn *Txn) (err error) { 290 | b.ResetTimer() 291 | defer b.StopTimer() 292 | for i := 0; i < b.N; i++ { 293 | err := benchmarkScanDBI(txn, dbi) 294 | if err != nil { 295 | return err 296 | } 297 | } 298 | 299 | return nil 300 | }) 301 | if err != nil { 302 | b.Error(err) 303 | return 304 | } 305 | } 306 | 307 | // like BenchmarkCursoreScanReadonly but txn.RawRead is set to true. 308 | func BenchmarkScan_raw_ro(b *testing.B) { 309 | initRandSource(b) 310 | env := setup(b) 311 | defer clean(env, b) 312 | 313 | dbi := openBenchDBI(b, env) 314 | 315 | rc := newRandSourceCursor() 316 | _, err := populateBenchmarkDB(env, dbi, &rc) 317 | if err != nil { 318 | b.Errorf("populate db: %v", err) 319 | return 320 | } 321 | 322 | err = env.View(func(txn *Txn) (err error) { 323 | txn.RawRead = true 324 | 325 | b.ResetTimer() 326 | defer b.StopTimer() 327 | for i := 0; i < b.N; i++ { 328 | err := benchmarkScanDBI(txn, dbi) 329 | if err != nil { 330 | return err 331 | } 332 | } 333 | 334 | return nil 335 | }) 336 | if err != nil { 337 | b.Errorf("benchmark: %v", err) 338 | return 339 | } 340 | } 341 | 342 | // populateBenchmarkDB fills env with data. 343 | // 344 | // populateBenchmarkDB calls env.SetMapSize and must not be called concurrent 345 | // with other transactions. 346 | func populateBenchmarkDB(env *Env, dbi DBI, rc *randSourceCursor) ([][]byte, error) { 347 | var ps [][]byte 348 | 349 | err := env.SetMapSize(benchDBMapSize) 350 | if err != nil { 351 | return nil, err 352 | } 353 | 354 | err = env.Update(func(txn *Txn) (err error) { 355 | for i := 0; i < benchDBNumKeys; i++ { 356 | k := makeBenchDBKey(rc) 357 | v := makeBenchDBVal(rc) 358 | err := txn.Put(dbi, k, v, 0) 359 | ps = append(ps, k, v) 360 | if err != nil { 361 | return err 362 | } 363 | } 364 | return nil 365 | }) 366 | if err != nil { 367 | return nil, err 368 | } 369 | return ps, nil 370 | } 371 | 372 | func benchmarkScanDBI(txn *Txn, dbi DBI) error { 373 | cur, err := txn.OpenCursor(dbi) 374 | if err != nil { 375 | return err 376 | } 377 | defer cur.Close() 378 | 379 | var count int64 380 | for { 381 | _, _, err := cur.Get(nil, nil, Next) 382 | if IsNotFound(err) { 383 | return nil 384 | } 385 | if err != nil { 386 | return err 387 | } 388 | count++ 389 | } 390 | } 391 | 392 | func openBenchDBI(b *testing.B, env *Env) DBI { 393 | var dbi DBI 394 | err := env.Update(func(txn *Txn) (err error) { 395 | dbi, err = txn.OpenDBI("benchmark", Create) 396 | return err 397 | }) 398 | if err != nil { 399 | b.Errorf("unable to open benchmark database") 400 | } 401 | return dbi 402 | } 403 | 404 | func teardownBenchDB(b *testing.B, env *Env, path string) { 405 | env.Close() 406 | os.RemoveAll(path) 407 | } 408 | 409 | func randBytes(n int) []byte { 410 | p := make([]byte, n) 411 | crand.Read(p) 412 | return p 413 | } 414 | 415 | func bMust(b *testing.B, err error, action string) { 416 | if err != nil { 417 | b.Fatalf("error %s: %v", action, err) 418 | } 419 | } 420 | 421 | const randSourceSize = 10 << 20 // size of the 'entropy pool' for random byte generation. 422 | const benchDBMapSize = 100 << 20 // size of a benchmark db memory map 423 | const benchDBNumKeys = 1 << 12 // number of keys to store in benchmark databases 424 | const benchDBMaxKeyLen = 30 // maximum length for database keys (size is limited by MDB) 425 | const benchDBMaxValLen = 4096 // maximum lengh for database values 426 | 427 | func makeBenchDBKey(c *randSourceCursor) []byte { 428 | return c.NBytes(rand.Intn(benchDBMaxKeyLen) + 1) 429 | } 430 | 431 | func makeBenchDBVal(c *randSourceCursor) []byte { 432 | return c.NBytes(rand.Intn(benchDBMaxValLen) + 1) 433 | } 434 | 435 | // holds a bunch of random bytes so repeated generation of 'random' slices is 436 | // cheap. acts as a ring which can be read from (although doesn't implement io.Reader). 437 | var _initRand int32 438 | var randSource [randSourceSize]byte 439 | 440 | func initRandSource(b *testing.B) { 441 | if atomic.AddInt32(&_initRand, 1) > 1 { 442 | return 443 | } 444 | b.Logf("initializing random source data") 445 | n, err := crand.Read(randSource[:]) 446 | bMust(b, err, "initializing random source") 447 | if n < len(randSource) { 448 | b.Fatalf("unable to read enough random source data %d", n) 449 | } 450 | } 451 | 452 | // acts as a simple byte slice generator. 453 | type randSourceCursor int 454 | 455 | func newRandSourceCursor() randSourceCursor { 456 | i := rand.Intn(randSourceSize) 457 | return randSourceCursor(i) 458 | } 459 | 460 | func (c *randSourceCursor) NBytes(n int) []byte { 461 | i := int(*c) 462 | if n >= randSourceSize { 463 | panic("rand size too big") 464 | } 465 | *c = (*c + randSourceCursor(n)) % randSourceSize 466 | _n := i + n - randSourceSize 467 | if _n > 0 { 468 | p := make([]byte, n) 469 | m := copy(p, randSource[i:]) 470 | copy(p[m:], randSource[:]) 471 | return p 472 | } 473 | return randSource[i : i+n] 474 | } 475 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/cursor.go: -------------------------------------------------------------------------------- 1 | package lmdb 2 | 3 | /* 4 | #include 5 | #include 6 | #include "lmdb.h" 7 | #include "lmdbgo.h" 8 | */ 9 | import "C" 10 | import ( 11 | "runtime" 12 | "unsafe" 13 | ) 14 | 15 | // These flags are used exclusively for Cursor.Get. 16 | const ( 17 | // Flags for Cursor.Get 18 | // 19 | // See MDB_cursor_op. 20 | 21 | First = C.MDB_FIRST // The first item. 22 | FirstDup = C.MDB_FIRST_DUP // The first value of current key (DupSort). 23 | GetBoth = C.MDB_GET_BOTH // Get the key as well as the value (DupSort). 24 | GetBothRange = C.MDB_GET_BOTH_RANGE // Get the key and the nearsest value (DupSort). 25 | GetCurrent = C.MDB_GET_CURRENT // Get the key and value at the current position. 26 | GetMultiple = C.MDB_GET_MULTIPLE // Get up to a page dup values for key at current position (DupFixed). 27 | Last = C.MDB_LAST // Last item. 28 | LastDup = C.MDB_LAST_DUP // Position at last value of current key (DupSort). 29 | Next = C.MDB_NEXT // Next value. 30 | NextDup = C.MDB_NEXT_DUP // Next value of the current key (DupSort). 31 | NextMultiple = C.MDB_NEXT_MULTIPLE // Get key and up to a page of values from the next cursor position (DupFixed). 32 | NextNoDup = C.MDB_NEXT_NODUP // The first value of the next key (DupSort). 33 | Prev = C.MDB_PREV // The previous item. 34 | PrevDup = C.MDB_PREV_DUP // The previous item of the current key (DupSort). 35 | PrevNoDup = C.MDB_PREV_NODUP // The last data item of the previous key (DupSort). 36 | Set = C.MDB_SET // The specified key. 37 | SetKey = C.MDB_SET_KEY // Get key and data at the specified key. 38 | SetRange = C.MDB_SET_RANGE // The first key no less than the specified key. 39 | ) 40 | 41 | // The MDB_MULTIPLE and MDB_RESERVE flags are special and do not fit the 42 | // calling pattern of other calls to Put. They are not exported because they 43 | // require special methods, PutMultiple and PutReserve in which the flag is 44 | // implied and does not need to be passed. 45 | const ( 46 | // Flags for Txn.Put and Cursor.Put. 47 | // 48 | // See mdb_put and mdb_cursor_put. 49 | 50 | Current = C.MDB_CURRENT // Replace the item at the current key position (Cursor only) 51 | NoDupData = C.MDB_NODUPDATA // Store the key-value pair only if key is not present (DupSort). 52 | NoOverwrite = C.MDB_NOOVERWRITE // Store a new key-value pair only if key is not present. 53 | Append = C.MDB_APPEND // Append an item to the database. 54 | AppendDup = C.MDB_APPENDDUP // Append an item to the database (DupSort). 55 | ) 56 | 57 | // Cursor operates on data inside a transaction and holds a position in the 58 | // database. 59 | // 60 | // See MDB_cursor. 61 | type Cursor struct { 62 | txn *Txn 63 | _c *C.MDB_cursor 64 | } 65 | 66 | func openCursor(txn *Txn, db DBI) (*Cursor, error) { 67 | c := &Cursor{txn: txn} 68 | ret := C.mdb_cursor_open(txn._txn, C.MDB_dbi(db), &c._c) 69 | if ret != success { 70 | return nil, operrno("mdb_cursor_open", ret) 71 | } 72 | return c, nil 73 | } 74 | 75 | // Renew associates readonly cursor with txn. 76 | // 77 | // See mdb_cursor_renew. 78 | func (c *Cursor) Renew(txn *Txn) error { 79 | ret := C.mdb_cursor_renew(txn._txn, c._c) 80 | err := operrno("mdb_cursor_renew", ret) 81 | if err != nil { 82 | return err 83 | } 84 | c.txn = txn 85 | return nil 86 | } 87 | 88 | func (c *Cursor) close() bool { 89 | if c._c != nil { 90 | if c.txn._txn == nil && !c.txn.readonly { 91 | // the cursor has already been released by LMDB. 92 | } else { 93 | C.mdb_cursor_close(c._c) 94 | } 95 | c.txn = nil 96 | c._c = nil 97 | return true 98 | } 99 | return false 100 | } 101 | 102 | // Close the cursor handle and clear the finalizer on c. Cursors belonging to 103 | // write transactions are closed automatically when the transaction is 104 | // terminated. 105 | // 106 | // See mdb_cursor_close. 107 | func (c *Cursor) Close() { 108 | if c.close() { 109 | runtime.SetFinalizer(c, nil) 110 | } 111 | } 112 | 113 | // Txn returns the cursor's transaction. 114 | func (c *Cursor) Txn() *Txn { 115 | return c.txn 116 | } 117 | 118 | // DBI returns the cursor's database handle. If c has been closed than an 119 | // invalid DBI is returned. 120 | func (c *Cursor) DBI() DBI { 121 | // dbiInvalid is an invalid DBI (the max value for the type). it shouldn't 122 | // be possible to create a database handle with value dbiInvalid because 123 | // the process address space would be exhausted. it is also impractical to 124 | // have many open databases in an environment. 125 | const dbiInvalid = ^DBI(0) 126 | 127 | // mdb_cursor_dbi segfaults when passed a nil value 128 | if c._c == nil { 129 | return dbiInvalid 130 | } 131 | return DBI(C.mdb_cursor_dbi(c._c)) 132 | } 133 | 134 | // Get retrieves items from the database. If c.Txn().RawRead is true the slices 135 | // returned by Get reference readonly sections of memory that must not be 136 | // accessed after the transaction has terminated. 137 | // 138 | // Get ignores setval if setkey is empty. 139 | // 140 | // See mdb_cursor_get. 141 | func (c *Cursor) Get(setkey, setval []byte, op uint) (key, val []byte, err error) { 142 | switch { 143 | case len(setkey) == 0: 144 | err = c.getVal0(op) 145 | case len(setval) == 0: 146 | err = c.getVal1(setkey, op) 147 | default: 148 | err = c.getVal2(setkey, setval, op) 149 | } 150 | if err != nil { 151 | *c.txn.key = C.MDB_val{} 152 | *c.txn.val = C.MDB_val{} 153 | return nil, nil, err 154 | } 155 | k := c.txn.bytes(c.txn.key) 156 | v := c.txn.bytes(c.txn.val) 157 | *c.txn.key = C.MDB_val{} 158 | *c.txn.val = C.MDB_val{} 159 | return k, v, nil 160 | } 161 | 162 | // getVal0 retrieves items from the database without using given key or value 163 | // data for reference (Next, First, Last, etc). 164 | // 165 | // See mdb_cursor_get. 166 | func (c *Cursor) getVal0(op uint) error { 167 | ret := C.mdb_cursor_get(c._c, c.txn.key, c.txn.val, C.MDB_cursor_op(op)) 168 | return operrno("mdb_cursor_get", ret) 169 | } 170 | 171 | // getVal1 retrieves items from the database using key data for reference 172 | // (Set, SetRange, etc). 173 | // 174 | // See mdb_cursor_get. 175 | func (c *Cursor) getVal1(setkey []byte, op uint) error { 176 | ret := C.lmdbgo_mdb_cursor_get1( 177 | c._c, 178 | (*C.char)(unsafe.Pointer(&setkey[0])), C.size_t(len(setkey)), 179 | c.txn.key, c.txn.val, 180 | C.MDB_cursor_op(op), 181 | ) 182 | return operrno("mdb_cursor_get", ret) 183 | } 184 | 185 | // getVal2 retrieves items from the database using key and value data for 186 | // reference (GetBoth, GetBothRange, etc). 187 | // 188 | // See mdb_cursor_get. 189 | func (c *Cursor) getVal2(setkey, setval []byte, op uint) error { 190 | ret := C.lmdbgo_mdb_cursor_get2( 191 | c._c, 192 | (*C.char)(unsafe.Pointer(&setkey[0])), C.size_t(len(setkey)), 193 | (*C.char)(unsafe.Pointer(&setval[0])), C.size_t(len(setval)), 194 | c.txn.key, c.txn.val, 195 | C.MDB_cursor_op(op), 196 | ) 197 | return operrno("mdb_cursor_get", ret) 198 | } 199 | 200 | func (c *Cursor) putNilKey(flags uint) error { 201 | ret := C.lmdbgo_mdb_cursor_put2(c._c, nil, 0, nil, 0, C.uint(flags)) 202 | return operrno("mdb_cursor_put", ret) 203 | } 204 | 205 | // Put stores an item in the database. 206 | // 207 | // See mdb_cursor_put. 208 | func (c *Cursor) Put(key, val []byte, flags uint) error { 209 | if len(key) == 0 { 210 | return c.putNilKey(flags) 211 | } 212 | vn := len(val) 213 | if vn == 0 { 214 | val = []byte{0} 215 | } 216 | ret := C.lmdbgo_mdb_cursor_put2( 217 | c._c, 218 | (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key)), 219 | (*C.char)(unsafe.Pointer(&val[0])), C.size_t(len(val)), 220 | C.uint(flags), 221 | ) 222 | return operrno("mdb_cursor_put", ret) 223 | } 224 | 225 | // PutReserve returns a []byte of length n that can be written to, potentially 226 | // avoiding a memcopy. The returned byte slice is only valid in txn's thread, 227 | // before it has terminated. 228 | func (c *Cursor) PutReserve(key []byte, n int, flags uint) ([]byte, error) { 229 | if len(key) == 0 { 230 | return nil, c.putNilKey(flags) 231 | } 232 | 233 | c.txn.val.mv_size = C.size_t(n) 234 | ret := C.lmdbgo_mdb_cursor_put1( 235 | c._c, 236 | (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key)), 237 | c.txn.val, 238 | C.uint(flags|C.MDB_RESERVE), 239 | ) 240 | err := operrno("mdb_cursor_put", ret) 241 | if err != nil { 242 | *c.txn.val = C.MDB_val{} 243 | return nil, err 244 | } 245 | b := getBytes(c.txn.val) 246 | *c.txn.val = C.MDB_val{} 247 | return b, nil 248 | } 249 | 250 | // PutMulti stores a set of contiguous items with stride size under key. 251 | // PutMulti panics if len(page) is not a multiple of stride. The cursor's 252 | // database must be DupFixed and DupSort. 253 | // 254 | // See mdb_cursor_put. 255 | func (c *Cursor) PutMulti(key []byte, page []byte, stride int, flags uint) error { 256 | if len(key) == 0 { 257 | return c.putNilKey(flags) 258 | } 259 | if len(page) == 0 { 260 | page = []byte{0} 261 | } 262 | 263 | vn := WrapMulti(page, stride).Len() 264 | ret := C.lmdbgo_mdb_cursor_putmulti( 265 | c._c, 266 | (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key)), 267 | (*C.char)(unsafe.Pointer(&page[0])), C.size_t(vn), C.size_t(stride), 268 | C.uint(flags|C.MDB_MULTIPLE), 269 | ) 270 | return operrno("mdb_cursor_put", ret) 271 | } 272 | 273 | // Del deletes the item referred to by the cursor from the database. 274 | // 275 | // See mdb_cursor_del. 276 | func (c *Cursor) Del(flags uint) error { 277 | ret := C.mdb_cursor_del(c._c, C.uint(flags)) 278 | return operrno("mdb_cursor_del", ret) 279 | } 280 | 281 | // Count returns the number of duplicates for the current key. 282 | // 283 | // See mdb_cursor_count. 284 | func (c *Cursor) Count() (uint64, error) { 285 | var _size C.size_t 286 | ret := C.mdb_cursor_count(c._c, &_size) 287 | if ret != success { 288 | return 0, operrno("mdb_cursor_count", ret) 289 | } 290 | return uint64(_size), nil 291 | } 292 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/env_test.go: -------------------------------------------------------------------------------- 1 | package lmdb 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "sync" 10 | "syscall" 11 | "testing" 12 | ) 13 | 14 | func TestEnv_Path_notOpen(t *testing.T) { 15 | env, err := NewEnv() 16 | if err != nil { 17 | t.Fatalf("create: %v", err) 18 | } 19 | defer env.Close() 20 | 21 | // before Open the Path method returns "" and a non-nil error. 22 | path, err := env.Path() 23 | if err == nil { 24 | t.Errorf("no error returned before Open") 25 | } 26 | if path != "" { 27 | t.Errorf("non-zero path returned before Open") 28 | } 29 | } 30 | 31 | func TestEnv_Path(t *testing.T) { 32 | env, err := NewEnv() 33 | if err != nil { 34 | t.Fatalf("create: %v", err) 35 | } 36 | 37 | // open an environment 38 | dir, err := ioutil.TempDir("", "mdb_test") 39 | if err != nil { 40 | t.Fatalf("tempdir: %v", err) 41 | } 42 | defer os.RemoveAll(dir) 43 | 44 | err = env.Open(dir, 0, 0644) 45 | defer env.Close() 46 | if err != nil { 47 | t.Errorf("open: %v", err) 48 | } 49 | path, err := env.Path() 50 | if err != nil { 51 | t.Errorf("path: %v", err) 52 | } 53 | if path != dir { 54 | t.Errorf("path: %q (!= %q)", path, dir) 55 | } 56 | } 57 | 58 | func TestEnv_Open_notExist(t *testing.T) { 59 | env, err := NewEnv() 60 | if err != nil { 61 | t.Fatalf("create: %s", err) 62 | } 63 | defer env.Close() 64 | 65 | // ensure that opening a non-existent path fails. 66 | err = env.Open("/path/does/not/exist/aoeu", 0, 0664) 67 | if !IsNotExist(err) { 68 | t.Errorf("open: %v", err) 69 | } 70 | } 71 | 72 | func TestEnv_Open(t *testing.T) { 73 | env, err := NewEnv() 74 | if err != nil { 75 | t.Error(err) 76 | return 77 | } 78 | defer func() { 79 | err := env.Close() 80 | if err != nil { 81 | t.Error(err) 82 | } 83 | }() 84 | 85 | // open an environment at a temporary path. 86 | path, err := ioutil.TempDir("", "mdb_test") 87 | if err != nil { 88 | t.Fatalf("tempdir: %v", err) 89 | } 90 | defer os.RemoveAll(path) 91 | err = env.Open(path, 0, 0664) 92 | if err != nil { 93 | t.Errorf("open: %s", err) 94 | } 95 | } 96 | 97 | func TestEnv_FD(t *testing.T) { 98 | env, err := NewEnv() 99 | if err != nil { 100 | t.Error(err) 101 | return 102 | } 103 | defer func() { 104 | err := env.Close() 105 | if err != nil { 106 | t.Error(err) 107 | } 108 | }() 109 | 110 | fd, err := env.FD() 111 | if err != errNotOpen { 112 | t.Errorf("fd: %x (%v)", fd, err) 113 | } 114 | 115 | // open an environment at a temporary path. 116 | path, err := ioutil.TempDir("", "mdb_test") 117 | if err != nil { 118 | t.Fatalf("tempdir: %v", err) 119 | } 120 | defer os.RemoveAll(path) 121 | err = env.Open(path, 0, 0664) 122 | if err != nil { 123 | t.Errorf("open: %s", err) 124 | } 125 | 126 | fd, err = env.FD() 127 | if err != nil { 128 | t.Errorf("fd error: %v", err) 129 | } 130 | if fd == 0 { 131 | t.Errorf("fd: %x", fd) 132 | } 133 | } 134 | 135 | func TestEnv_Flags(t *testing.T) { 136 | env := setup(t) 137 | defer clean(env, t) 138 | 139 | flags, err := env.Flags() 140 | if err != nil { 141 | t.Error(err) 142 | return 143 | } 144 | 145 | if flags&NoTLS == 0 { 146 | t.Errorf("NoTLS is not set") 147 | } 148 | if flags&NoSync != 0 { 149 | t.Errorf("NoSync is set") 150 | } 151 | 152 | err = env.SetFlags(NoSync) 153 | if err != nil { 154 | t.Error(err) 155 | } 156 | 157 | flags, err = env.Flags() 158 | if err != nil { 159 | t.Error(err) 160 | } 161 | if flags&NoSync == 0 { 162 | t.Error("NoSync is not set") 163 | } 164 | 165 | err = env.UnsetFlags(NoSync) 166 | if err != nil { 167 | t.Error(err) 168 | } 169 | 170 | flags, err = env.Flags() 171 | if err != nil { 172 | t.Error(err) 173 | } 174 | if flags&NoSync != 0 { 175 | t.Error("NoSync is set") 176 | } 177 | } 178 | 179 | func TestEnv_SetMaxReader(t *testing.T) { 180 | dir, err := ioutil.TempDir("", "test-env-setmaxreaders-") 181 | if err != nil { 182 | t.Fatal(err) 183 | } 184 | defer os.RemoveAll(dir) 185 | 186 | env, err := NewEnv() 187 | if err != nil { 188 | t.Error(err) 189 | } 190 | 191 | maxreaders := 5 192 | err = env.SetMaxReaders(maxreaders) 193 | if err != nil { 194 | t.Error(err) 195 | } 196 | _maxreaders, err := env.MaxReaders() 197 | if err != nil { 198 | t.Error(err) 199 | } 200 | if _maxreaders != maxreaders { 201 | t.Errorf("unexpected MaxReaders: %v (!= %v)", _maxreaders, maxreaders) 202 | } 203 | 204 | err = env.Open(dir, 0, 0644) 205 | defer env.Close() 206 | if err != nil { 207 | env.Close() 208 | t.Error(err) 209 | } 210 | 211 | err = env.SetMaxReaders(126) 212 | if !IsErrnoSys(err, syscall.EINVAL) { 213 | t.Errorf("unexpected error: %v (!= %v)", err, syscall.EINVAL) 214 | } 215 | _maxreaders, err = env.MaxReaders() 216 | if err != nil { 217 | t.Error(err) 218 | } 219 | if _maxreaders != maxreaders { 220 | t.Errorf("unexpected MaxReaders: %v (!= %v)", _maxreaders, maxreaders) 221 | } 222 | } 223 | 224 | func TestEnv_SetMapSize(t *testing.T) { 225 | env := setup(t) 226 | defer clean(env, t) 227 | 228 | const minsize = 100 << 20 // 100MB 229 | err := env.SetMapSize(minsize) 230 | if err != nil { 231 | t.Error(err) 232 | } 233 | 234 | err = env.Update(func(txn *Txn) (err error) { 235 | return nil 236 | }) 237 | if err != nil { 238 | t.Error(err) 239 | } 240 | 241 | info, err := env.Info() 242 | if err != nil { 243 | t.Error(err) 244 | } else if info.MapSize < minsize { 245 | t.Errorf("unexpected mapsize: %v (< %v)", info.MapSize, minsize) 246 | } 247 | } 248 | 249 | func TestEnv_ReaderList(t *testing.T) { 250 | env := setup(t) 251 | defer clean(env, t) 252 | 253 | var numreaders = 2 254 | 255 | var fin sync.WaitGroup 256 | defer fin.Wait() 257 | ready := make(chan struct{}) 258 | done := make(chan struct{}) 259 | defer close(done) 260 | 261 | t.Logf("starting") 262 | 263 | for i := 0; i < numreaders; i++ { 264 | fin.Add(1) 265 | go func(i int) { 266 | defer fin.Done() 267 | err := env.View(func(txn *Txn) (err error) { 268 | t.Logf("reader %v: ready", i) 269 | ready <- struct{}{} 270 | 271 | <-done 272 | t.Logf("reader %v: done", i) 273 | return nil 274 | }) 275 | if err != nil { 276 | t.Errorf("reader %d: %q", i, err) 277 | } 278 | }(i) 279 | 280 | // wait for each reader to become ready 281 | <-ready 282 | } 283 | 284 | var readers []string 285 | env.ReaderList(func(msg string) error { 286 | t.Logf("reader: %q", msg) 287 | readers = append(readers, msg) 288 | return nil 289 | }) 290 | if len(readers) != numreaders+1 { 291 | t.Errorf("unexpected reader list size: %d (!= %d)", len(readers), numreaders) 292 | } 293 | } 294 | 295 | func TestEnv_ReaderList_error(t *testing.T) { 296 | env := setup(t) 297 | defer clean(env, t) 298 | 299 | var numreaders = 2 300 | 301 | var fin sync.WaitGroup 302 | defer fin.Wait() 303 | ready := make(chan struct{}) 304 | done := make(chan struct{}) 305 | defer close(done) 306 | 307 | t.Logf("starting") 308 | 309 | for i := 0; i < numreaders; i++ { 310 | fin.Add(1) 311 | go func(i int) { 312 | defer fin.Done() 313 | err := env.View(func(txn *Txn) (err error) { 314 | t.Logf("reader %v: ready", i) 315 | ready <- struct{}{} 316 | 317 | <-done 318 | t.Logf("reader %v: done", i) 319 | return nil 320 | }) 321 | if err != nil { 322 | t.Errorf("reader %d: %q", i, err) 323 | } 324 | }(i) 325 | 326 | // wait for each reader to become ready 327 | <-ready 328 | } 329 | 330 | e := fmt.Errorf("testerror") 331 | var readers []string 332 | err := env.ReaderList(func(msg string) error { 333 | readers = append(readers, msg) 334 | return e 335 | }) 336 | if err == nil { 337 | t.Errorf("expected error") 338 | } 339 | if err != e { 340 | t.Errorf("unexpected error: %q (!= %q)", err, e) 341 | } 342 | if len(readers) != 1 { 343 | t.Errorf("unexpected reader list size: %d (!= %d)", len(readers), 1) 344 | } 345 | } 346 | 347 | func TestEnv_ReaderList_envInvalid(t *testing.T) { 348 | err := (&Env{}).ReaderList(func(msg string) error { 349 | t.Logf("%s", msg) 350 | return nil 351 | }) 352 | if err == nil { 353 | t.Errorf("expected error") 354 | } 355 | } 356 | 357 | func TestEnv_ReaderList_nilFunc(t *testing.T) { 358 | env, err := NewEnv() 359 | if err != nil { 360 | t.Fatal(err) 361 | } 362 | err = env.ReaderList(nil) 363 | if err == nil { 364 | t.Errorf("expected error") 365 | } 366 | } 367 | 368 | func TestEnv_ReaderCheck(t *testing.T) { 369 | env := setup(t) 370 | defer clean(env, t) 371 | 372 | numDead, err := env.ReaderCheck() 373 | if err != nil { 374 | t.Error(err) 375 | } 376 | if numDead != 0 { 377 | t.Errorf("unexpected dead readers: %v (!= %v)", numDead, 0) 378 | } 379 | } 380 | 381 | func TestEnv_Copy(t *testing.T) { 382 | testEnvCopy(t, 0, false, false) 383 | } 384 | 385 | func TestEnv_CopyFlags(t *testing.T) { 386 | testEnvCopy(t, CopyCompact, true, false) 387 | } 388 | 389 | func TestEnv_CopyFlags_zero(t *testing.T) { 390 | testEnvCopy(t, 0, true, false) 391 | } 392 | 393 | func TestEnv_CopyFD(t *testing.T) { 394 | testEnvCopy(t, 0, false, true) 395 | } 396 | 397 | func TestEnv_CopyFDFlags(t *testing.T) { 398 | testEnvCopy(t, CopyCompact, true, true) 399 | } 400 | 401 | func TestEnv_CopyFDFlags_zero(t *testing.T) { 402 | testEnvCopy(t, 0, true, true) 403 | } 404 | 405 | func testEnvCopy(t *testing.T, flags uint, useflags bool, usefd bool) { 406 | dircp, err := ioutil.TempDir("", "test-env-copy-") 407 | if err != nil { 408 | t.Fatal(err) 409 | } 410 | defer os.RemoveAll(dircp) 411 | 412 | var fd uintptr 413 | if usefd { 414 | path := filepath.Join(dircp, "data.mdb") 415 | f, err := os.Create(path) 416 | if err != nil { 417 | t.Error(err) 418 | return 419 | } 420 | fd = f.Fd() 421 | defer f.Close() 422 | } 423 | 424 | env := setup(t) 425 | defer clean(env, t) 426 | 427 | item := struct{ k, v []byte }{ 428 | []byte("k0"), 429 | []byte("v0"), 430 | } 431 | 432 | err = env.Update(func(txn *Txn) (err error) { 433 | db, err := txn.OpenRoot(0) 434 | if err != nil { 435 | return err 436 | } 437 | return txn.Put(db, item.k, item.v, 0) 438 | }) 439 | if err != nil { 440 | t.Error(err) 441 | } 442 | 443 | switch { 444 | case usefd && useflags: 445 | err = env.CopyFDFlag(fd, flags) 446 | case usefd && !useflags: 447 | err = env.CopyFD(fd) 448 | case !usefd && useflags: 449 | err = env.CopyFlag(dircp, flags) 450 | case !usefd && !useflags: 451 | err = env.Copy(dircp) 452 | } 453 | if err != nil { 454 | t.Error(err) 455 | } 456 | 457 | envcp, err := NewEnv() 458 | if err != nil { 459 | t.Error(err) 460 | } 461 | err = envcp.Open(dircp, 0, 0644) 462 | defer envcp.Close() 463 | if err != nil { 464 | t.Error(err) 465 | return 466 | } 467 | 468 | err = envcp.View(func(txn *Txn) (err error) { 469 | db, err := txn.OpenRoot(0) 470 | if err != nil { 471 | return err 472 | } 473 | v, err := txn.Get(db, item.k) 474 | if err != nil { 475 | return err 476 | } 477 | if !bytes.Equal(v, item.v) { 478 | return fmt.Errorf("unexpected value: %q (!= %q)", v, "v0") 479 | } 480 | return nil 481 | }) 482 | if err != nil { 483 | t.Error(err) 484 | } 485 | } 486 | 487 | func TestEnv_Sync(t *testing.T) { 488 | env := setupFlags(t, NoSync) 489 | defer clean(env, t) 490 | 491 | item := struct{ k, v []byte }{[]byte("k0"), []byte("v0")} 492 | 493 | err := env.Update(func(txn *Txn) (err error) { 494 | db, err := txn.OpenRoot(0) 495 | if err != nil { 496 | return err 497 | } 498 | return txn.Put(db, item.k, item.v, 0) 499 | }) 500 | if err != nil { 501 | t.Error(err) 502 | } 503 | 504 | err = env.Sync(true) 505 | if err != nil { 506 | t.Error(err) 507 | } 508 | } 509 | 510 | func setup(t T) *Env { 511 | return setupFlags(t, 0) 512 | } 513 | 514 | func setupFlags(t T, flags uint) *Env { 515 | env, err := NewEnv() 516 | if err != nil { 517 | t.Fatalf("env: %s", err) 518 | } 519 | path, err := ioutil.TempDir("", "mdb_test") 520 | if err != nil { 521 | t.Fatalf("tempdir: %v", err) 522 | } 523 | err = os.MkdirAll(path, 0770) 524 | if err != nil { 525 | t.Fatalf("mkdir: %s", path) 526 | } 527 | err = env.SetMaxDBs(64 << 10) 528 | if err != nil { 529 | t.Fatalf("setmaxdbs: %v", err) 530 | } 531 | err = env.Open(path, flags, 0664) 532 | if err != nil { 533 | t.Fatalf("open: %s", err) 534 | } 535 | 536 | return env 537 | } 538 | 539 | type T interface { 540 | Errorf(format string, vals ...interface{}) 541 | Fatalf(format string, vals ...interface{}) 542 | } 543 | 544 | func clean(env *Env, t T) { 545 | path, err := env.Path() 546 | if err != nil { 547 | t.Errorf("path: %v", err) 548 | } 549 | err = env.Close() 550 | if err != nil { 551 | t.Errorf("close: %s", err) 552 | } 553 | if path != "" { 554 | err = os.RemoveAll(path) 555 | if err != nil { 556 | t.Errorf("remove: %v", err) 557 | } 558 | } 559 | } 560 | 561 | func TestEnvCopy(t *testing.T) { 562 | env := setup(t) 563 | defer clean(env, t) 564 | } 565 | 566 | func TestEnv_MaxKeySize(t *testing.T) { 567 | env := setup(t) 568 | defer clean(env, t) 569 | 570 | n := env.MaxKeySize() 571 | if n <= 0 { 572 | t.Errorf("invaild maxkeysize: %d", n) 573 | } 574 | } 575 | 576 | func TestEnv_MaxKeySize_nil(t *testing.T) { 577 | var env *Env 578 | n := env.MaxKeySize() 579 | if n <= 0 { 580 | t.Errorf("invaild maxkeysize: %d", n) 581 | } 582 | t.Logf("mdb_env_get_maxkeysize: %d", n) 583 | } 584 | 585 | func TestEnv_CloseDBI(t *testing.T) { 586 | env := setup(t) 587 | defer clean(env, t) 588 | 589 | const numdb = 1000 590 | for i := 0; i < numdb; i++ { 591 | dbname := fmt.Sprintf("db%d", i) 592 | 593 | var dbi DBI 594 | err := env.Update(func(txn *Txn) (err error) { 595 | dbi, err = txn.CreateDBI(dbname) 596 | return err 597 | }) 598 | if err != nil { 599 | t.Errorf("%s", err) 600 | } 601 | 602 | env.CloseDBI(dbi) 603 | } 604 | 605 | stat, err := env.Stat() 606 | if err != nil { 607 | t.Errorf("%s", err) 608 | return 609 | } 610 | 611 | if stat.Entries != numdb { 612 | t.Errorf("unexpected entries: %d (not %d)", stat.Entries, numdb) 613 | } 614 | } 615 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/error.go: -------------------------------------------------------------------------------- 1 | package lmdb 2 | 3 | /* 4 | #include "lmdb.h" 5 | */ 6 | import "C" 7 | 8 | import ( 9 | "os" 10 | "syscall" 11 | ) 12 | 13 | // OpError is an error returned by the C API. Not all errors returned by 14 | // lmdb-go have type OpError but typically they do. The Errno field will 15 | // either have type Errno or syscall.Errno. 16 | type OpError struct { 17 | Op string 18 | Errno error 19 | } 20 | 21 | // Error implements the error interface. 22 | func (err *OpError) Error() string { 23 | return err.Op + ": " + err.Errno.Error() 24 | } 25 | 26 | // The most common error codes do not need to be handled explicity. Errors can 27 | // be checked through helper functions IsNotFound, IsMapFull, etc, Otherwise 28 | // they should be checked using the IsErrno function instead of direct 29 | // comparison because they will typically be wrapped with an OpError. 30 | const ( 31 | // Error codes defined by LMDB. See the list of LMDB return codes for more 32 | // information about each 33 | // 34 | // http://symas.com/mdb/doc/group__errors.html 35 | 36 | KeyExist Errno = C.MDB_KEYEXIST 37 | NotFound Errno = C.MDB_NOTFOUND 38 | PageNotFound Errno = C.MDB_PAGE_NOTFOUND 39 | Corrupted Errno = C.MDB_CORRUPTED 40 | Panic Errno = C.MDB_PANIC 41 | VersionMismatch Errno = C.MDB_VERSION_MISMATCH 42 | Invalid Errno = C.MDB_INVALID 43 | MapFull Errno = C.MDB_MAP_FULL 44 | DBsFull Errno = C.MDB_DBS_FULL 45 | ReadersFull Errno = C.MDB_READERS_FULL 46 | TLSFull Errno = C.MDB_TLS_FULL 47 | TxnFull Errno = C.MDB_TXN_FULL 48 | CursorFull Errno = C.MDB_CURSOR_FULL 49 | PageFull Errno = C.MDB_PAGE_FULL 50 | MapResized Errno = C.MDB_MAP_RESIZED 51 | Incompatible Errno = C.MDB_INCOMPATIBLE 52 | BadRSlot Errno = C.MDB_BAD_RSLOT 53 | BadTxn Errno = C.MDB_BAD_TXN 54 | BadValSize Errno = C.MDB_BAD_VALSIZE 55 | BadDBI Errno = C.MDB_BAD_DBI 56 | ) 57 | 58 | // Errno is an error type that represents the (unique) errno values defined by 59 | // LMDB. Other errno values (such as EINVAL) are represented with type 60 | // syscall.Errno. On Windows, LMDB return codes are translated into portable 61 | // syscall.Errno constants (e.g. syscall.EINVAL, syscall.EACCES, etc.). 62 | // 63 | // Most often helper functions such as IsNotFound may be used instead of 64 | // dealing with Errno values directly. 65 | // 66 | // lmdb.IsNotFound(err) 67 | // lmdb.IsErrno(err, lmdb.TxnFull) 68 | // lmdb.IsErrnoSys(err, syscall.EINVAL) 69 | // lmdb.IsErrnoFn(err, os.IsPermission) 70 | type Errno C.int 71 | 72 | // minimum and maximum values produced for the Errno type. syscall.Errnos of 73 | // other values may still be produced. 74 | const minErrno, maxErrno C.int = C.MDB_KEYEXIST, C.MDB_LAST_ERRCODE 75 | 76 | func (e Errno) Error() string { 77 | return C.GoString(C.mdb_strerror(C.int(e))) 78 | } 79 | 80 | // _operrno is for use by tests that can't import C 81 | func _operrno(op string, ret int) error { 82 | return operrno(op, C.int(ret)) 83 | } 84 | 85 | // IsNotFound returns true if the key requested in Txn.Get or Cursor.Get does 86 | // not exist or if the Cursor reached the end of the database without locating 87 | // a value (EOF). 88 | func IsNotFound(err error) bool { 89 | return IsErrno(err, NotFound) 90 | } 91 | 92 | // IsNotExist returns true the path passed to the Env.Open method does not 93 | // exist. 94 | func IsNotExist(err error) bool { 95 | return IsErrnoFn(err, os.IsNotExist) 96 | } 97 | 98 | // IsMapFull returns true if the environment map size has been reached. 99 | func IsMapFull(err error) bool { 100 | return IsErrno(err, MapFull) 101 | } 102 | 103 | // IsMapResized returns true if the environment has grown too large for the 104 | // current map after being resized by another process. 105 | func IsMapResized(err error) bool { 106 | return IsErrno(err, MapResized) 107 | } 108 | 109 | // IsErrno returns true if err's errno is the given errno. 110 | func IsErrno(err error, errno Errno) bool { 111 | return IsErrnoFn(err, func(err error) bool { return err == errno }) 112 | } 113 | 114 | // IsErrnoSys returns true if err's errno is the given errno. 115 | func IsErrnoSys(err error, errno syscall.Errno) bool { 116 | return IsErrnoFn(err, func(err error) bool { return err == errno }) 117 | } 118 | 119 | // IsErrnoFn calls fn on the error underlying err and returns the result. If 120 | // err is an *OpError then err.Errno is passed to fn. Otherwise err is passed 121 | // directly to fn. 122 | func IsErrnoFn(err error, fn func(error) bool) bool { 123 | if err == nil { 124 | return false 125 | } 126 | if err, ok := err.(*OpError); ok { 127 | return fn(err.Errno) 128 | } 129 | return fn(err) 130 | } 131 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/error_test.go: -------------------------------------------------------------------------------- 1 | package lmdb 2 | 3 | import ( 4 | "fmt" 5 | "syscall" 6 | "testing" 7 | ) 8 | 9 | func TestErrno_Error(t *testing.T) { 10 | operr := &OpError{"testop", fmt.Errorf("testmsg")} 11 | msg := operr.Error() 12 | if msg != "testop: testmsg" { 13 | t.Errorf("message: %q", msg) 14 | } 15 | } 16 | 17 | func BenchmarkErrno_Error(b *testing.B) { 18 | for i := 0; i < b.N; i++ { 19 | for _, errno := range []error{ 20 | syscall.EINVAL, 21 | NotFound, 22 | MapResized, 23 | MapFull, 24 | } { 25 | operr := &OpError{"mdb_testop", errno} 26 | msg := operr.Error() 27 | if msg == "" { 28 | b.Fatal("empty message") 29 | } 30 | } 31 | 32 | } 33 | } 34 | func TestErrno(t *testing.T) { 35 | zeroerr := operrno("testop", 0) 36 | if zeroerr != nil { 37 | t.Errorf("errno(0) != nil: %#v", zeroerr) 38 | } 39 | syserr := _operrno("testop", int(syscall.EINVAL)) 40 | if syserr.(*OpError).Errno != syscall.EINVAL { // fails if error is Errno(syscall.EINVAL) 41 | t.Errorf("errno(syscall.EINVAL) != syscall.EINVAL: %#v", syserr) 42 | } 43 | mdberr := _operrno("testop", int(KeyExist)) 44 | if mdberr.(*OpError).Errno != KeyExist { 45 | t.Errorf("errno(ErrKeyExist) != ErrKeyExist: %#v", syserr) 46 | } 47 | } 48 | 49 | func TestIsErrno(t *testing.T) { 50 | err := NotFound 51 | if !IsErrno(err, err) { 52 | t.Errorf("expected match: %v", err) 53 | } 54 | 55 | operr := &OpError{ 56 | Op: "testop", 57 | Errno: err, 58 | } 59 | if !IsErrno(operr, err) { 60 | t.Errorf("expected match: %v", operr) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/error_unix.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package lmdb 4 | 5 | /* 6 | #include "lmdb.h" 7 | */ 8 | import "C" 9 | import "syscall" 10 | 11 | func operrno(op string, ret C.int) error { 12 | if ret == C.MDB_SUCCESS { 13 | return nil 14 | } 15 | if minErrno <= ret && ret <= maxErrno { 16 | return &OpError{Op: op, Errno: Errno(ret)} 17 | } 18 | return &OpError{Op: op, Errno: syscall.Errno(ret)} 19 | } 20 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/error_windows.go: -------------------------------------------------------------------------------- 1 | package lmdb 2 | 3 | /* 4 | #include 5 | #include "lmdb.h" 6 | */ 7 | import "C" 8 | import "syscall" 9 | 10 | func operrno(op string, ret C.int) error { 11 | if ret == C.MDB_SUCCESS { 12 | return nil 13 | } 14 | if minErrno <= ret && ret <= maxErrno { 15 | return &OpError{Op: op, Errno: Errno(ret)} 16 | } 17 | 18 | // translate C errors into corresponding syscall.Errno values so that 19 | // IsErrnoSys functions correctly, a kludge unknowning inherited from LMDB. 20 | // the errno in the returned OpError cannot be passed to C.mdb_strerror. 21 | // see the implementation of C.mdb_strerror for information about how the 22 | // following table was generated. 23 | var errno syscall.Errno 24 | switch ret { 25 | case C.ENOENT: 26 | errno = syscall.ENOENT /* 2, FILE_NOT_FOUND */ 27 | case C.EIO: 28 | errno = syscall.EIO /* 5, ACCESS_DENIED */ 29 | case C.ENOMEM: 30 | errno = syscall.ENOMEM /* 12, INVALID_ACCESS */ 31 | case C.EACCES: 32 | errno = syscall.EACCES /* 13, INVALID_DATA */ 33 | case C.EBUSY: 34 | errno = syscall.EBUSY /* 16, CURRENT_DIRECTORY */ 35 | case C.EINVAL: 36 | errno = syscall.EINVAL /* 22, BAD_COMMAND */ 37 | case C.ENOSPC: 38 | errno = syscall.ENOSPC /* 28, OUT_OF_PAPER */ 39 | default: 40 | errno = syscall.Errno(ret) 41 | } 42 | return &OpError{Op: op, Errno: errno} 43 | } 44 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/lmdb.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package lmdb provides bindings to the lmdb C API. The package bindings are 3 | fairly low level and are designed to provide a minimal interface that prevents 4 | misuse to a reasonable extent. When in doubt refer to the C documentation as a 5 | reference. 6 | 7 | http://symas.com/mdb/doc/group__mdb.html 8 | 9 | Environment 10 | 11 | An LMDB environment holds named databases (key-value stores). An environment 12 | is represented as one file on the filesystem (though often a corresponding lock 13 | file exists). 14 | 15 | LMDB recommends setting an environment's size as large as possible at the time 16 | of creation. On filesystems that support sparse files this should not 17 | adversely affect disk usage. Resizing an environment is possible but must be 18 | handled with care when concurrent access is involved. 19 | 20 | Databases 21 | 22 | A database in an LMDB environment is an ordered key-value store that holds 23 | arbitrary binary data. Typically the keys are unique but duplicate keys may be 24 | allowed (DupSort), in which case the values for each duplicate key are ordered. 25 | 26 | A single LMDB environment can have multiple named databases. But there is also 27 | a 'root' (unnamed) database that can be used to store data. Use caution 28 | storing data in the root database when named databases are in use. The root 29 | database serves as an index for named databases. 30 | 31 | A database is referenced by an opaque handle known as its DBI which must be 32 | opened inside a transaction with the OpenDBI or OpenRoot methods. DBIs may be 33 | closed but it is not required. Typically, applications acquire handles for all 34 | their databases immediately after opening an environment and retain them for 35 | the lifetime of the process. 36 | 37 | Transactions 38 | 39 | View (readonly) transactions in LMDB operate on a snapshot of the database at 40 | the time the transaction began. The number of simultaneously active view 41 | transactions is bounded and configured when the environment is initialized. 42 | 43 | Update (read-write) transactions are serialized in LMDB. Attempts to create 44 | update transactions block until a lock may be obtained. Update transactions 45 | can create subtransactions which may be rolled back independently from their 46 | parent. 47 | 48 | The lmdb package supplies managed and unmanaged transactions. Managed 49 | transactions do not require explicit calling of Abort/Commit and are provided 50 | through the Env methods Update, View, and RunTxn. The BeginTxn method on Env 51 | creates an unmanaged transaction but its use is not advised in most 52 | applications. 53 | */ 54 | package lmdb 55 | 56 | /* 57 | #cgo CFLAGS: -pthread -W -Wall -Wno-unused-parameter -Wno-format-extra-args -Wbad-function-cast -O2 -g 58 | #cgo linux,pwritev CFLAGS: -DMDB_USE_PWRITEV 59 | 60 | #include "lmdb.h" 61 | */ 62 | import "C" 63 | 64 | // Version return the major, minor, and patch version numbers of the LMDB C 65 | // library and a string representation of the version. 66 | // 67 | // See mdb_version. 68 | func Version() (major, minor, patch int, s string) { 69 | var maj, min, pat C.int 70 | verstr := C.mdb_version(&maj, &min, &pat) 71 | return int(maj), int(min), int(pat), C.GoString(verstr) 72 | } 73 | 74 | // VersionString returns a string representation of the LMDB C library version. 75 | // 76 | // See mdb_version. 77 | func VersionString() string { 78 | var maj, min, pat C.int 79 | verstr := C.mdb_version(&maj, &min, &pat) 80 | return C.GoString(verstr) 81 | } 82 | 83 | func cbool(b bool) C.int { 84 | if b { 85 | return 1 86 | } 87 | return 0 88 | } 89 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/lmdb_test.go: -------------------------------------------------------------------------------- 1 | package lmdb 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | func TestTest1(t *testing.T) { 11 | env, err := NewEnv() 12 | if err != nil { 13 | t.Fatalf("Cannot create environment: %s", err) 14 | } 15 | err = env.SetMapSize(10485760) 16 | if err != nil { 17 | t.Fatalf("Cannot set mapsize: %s", err) 18 | } 19 | path, err := ioutil.TempDir("", "mdb_test") 20 | if err != nil { 21 | t.Fatalf("Cannot create temporary directory") 22 | } 23 | err = os.MkdirAll(path, 0770) 24 | defer os.RemoveAll(path) 25 | if err != nil { 26 | t.Fatalf("Cannot create directory: %s", path) 27 | } 28 | err = env.Open(path, 0, 0664) 29 | defer env.Close() 30 | if err != nil { 31 | t.Fatalf("Cannot open environment: %s", err) 32 | } 33 | 34 | var db DBI 35 | numEntries := 10 36 | var data = map[string]string{} 37 | var key string 38 | var val string 39 | for i := 0; i < numEntries; i++ { 40 | key = fmt.Sprintf("Key-%d", i) 41 | val = fmt.Sprintf("Val-%d", i) 42 | data[key] = val 43 | } 44 | err = env.Update(func(txn *Txn) (err error) { 45 | db, err = txn.OpenRoot(0) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | for k, v := range data { 51 | err = txn.Put(db, []byte(k), []byte(v), NoOverwrite) 52 | if err != nil { 53 | return fmt.Errorf("put: %v", err) 54 | } 55 | } 56 | 57 | return nil 58 | }) 59 | if err != nil { 60 | t.Fatal(err) 61 | } 62 | 63 | stat, err := env.Stat() 64 | if err != nil { 65 | t.Fatalf("Cannot get stat %s", err) 66 | } else if stat.Entries != uint64(numEntries) { 67 | t.Errorf("Less entry in the database than expected: %d <> %d", stat.Entries, numEntries) 68 | } 69 | t.Logf("%#v", stat) 70 | 71 | err = env.View(func(txn *Txn) error { 72 | cursor, err := txn.OpenCursor(db) 73 | if err != nil { 74 | cursor.Close() 75 | return fmt.Errorf("cursor: %v", err) 76 | } 77 | var bkey, bval []byte 78 | var bNumVal int 79 | for { 80 | bkey, bval, err = cursor.Get(nil, nil, Next) 81 | if IsNotFound(err) { 82 | break 83 | } 84 | if err != nil { 85 | return fmt.Errorf("cursor get: %v", err) 86 | } 87 | bNumVal++ 88 | skey := string(bkey) 89 | sval := string(bval) 90 | t.Logf("Val: %s", sval) 91 | t.Logf("Key: %s", skey) 92 | var d string 93 | var ok bool 94 | if d, ok = data[skey]; !ok { 95 | return fmt.Errorf("cursor get: key does not exist %q", skey) 96 | } 97 | if d != sval { 98 | return fmt.Errorf("cursor get: value %q does not match %q", sval, d) 99 | } 100 | } 101 | if bNumVal != numEntries { 102 | t.Errorf("cursor iterated over %d entries when %d expected", bNumVal, numEntries) 103 | } 104 | cursor.Close() 105 | bval, err = txn.Get(db, []byte("Key-0")) 106 | if err != nil { 107 | return fmt.Errorf("get: %v", err) 108 | } 109 | if string(bval) != "Val-0" { 110 | return fmt.Errorf("get: value %q does not match %q", bval, "Val-0") 111 | } 112 | return nil 113 | }) 114 | if err != nil { 115 | t.Fatal(err) 116 | } 117 | } 118 | 119 | func TestVersion(t *testing.T) { 120 | maj, min, patch, str := Version() 121 | if maj < 0 || min < 0 || patch < 0 { 122 | t.Error("invalid version number: ", maj, min, patch) 123 | } 124 | if maj == 0 && min == 0 && patch == 0 { 125 | t.Error("invalid version number: ", maj, min, patch) 126 | } 127 | if str == "" { 128 | t.Error("empty version string") 129 | } 130 | 131 | str = VersionString() 132 | if str == "" { 133 | t.Error("empty version string") 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/lmdbgo.c: -------------------------------------------------------------------------------- 1 | /* lmdbgo.c 2 | * Helper utilities for github.com/bmatsuo/lmdb-go/lmdb 3 | * */ 4 | #include "lmdb.h" 5 | #include "lmdbgo.h" 6 | #include "_cgo_export.h" 7 | 8 | #define LMDBGO_SET_VAL(val, size, data) \ 9 | *(val) = (MDB_val){.mv_size = (size), .mv_data = (data)} 10 | 11 | int lmdbgo_mdb_msg_func_proxy(const char *msg, void *ctx) { 12 | // wrap msg and call the bridge function exported from lmdb.go. 13 | lmdbgo_ConstCString s; 14 | s.p = msg; 15 | return lmdbgoMDBMsgFuncBridge(s, (size_t)ctx); 16 | } 17 | 18 | int lmdbgo_mdb_reader_list(MDB_env *env, size_t ctx) { 19 | // list readers using a static proxy function that does dynamic dispatch on 20 | // ctx. 21 | if (ctx) 22 | return mdb_reader_list(env, &lmdbgo_mdb_msg_func_proxy, (void *)ctx); 23 | return mdb_reader_list(env, 0, (void *)ctx); 24 | } 25 | 26 | int lmdbgo_mdb_del(MDB_txn *txn, MDB_dbi dbi, char *kdata, size_t kn, char *vdata, size_t vn) { 27 | MDB_val key, val; 28 | LMDBGO_SET_VAL(&key, kn, kdata); 29 | LMDBGO_SET_VAL(&val, vn, vdata); 30 | return mdb_del(txn, dbi, &key, &val); 31 | } 32 | 33 | int lmdbgo_mdb_get(MDB_txn *txn, MDB_dbi dbi, char *kdata, size_t kn, MDB_val *val) { 34 | MDB_val key; 35 | LMDBGO_SET_VAL(&key, kn, kdata); 36 | return mdb_get(txn, dbi, &key, val); 37 | } 38 | 39 | int lmdbgo_mdb_put2(MDB_txn *txn, MDB_dbi dbi, char *kdata, size_t kn, char *vdata, size_t vn, unsigned int flags) { 40 | MDB_val key, val; 41 | LMDBGO_SET_VAL(&key, kn, kdata); 42 | LMDBGO_SET_VAL(&val, vn, vdata); 43 | return mdb_put(txn, dbi, &key, &val, flags); 44 | } 45 | 46 | int lmdbgo_mdb_put1(MDB_txn *txn, MDB_dbi dbi, char *kdata, size_t kn, MDB_val *val, unsigned int flags) { 47 | MDB_val key; 48 | LMDBGO_SET_VAL(&key, kn, kdata); 49 | return mdb_put(txn, dbi, &key, val, flags); 50 | } 51 | 52 | int lmdbgo_mdb_cursor_put2(MDB_cursor *cur, char *kdata, size_t kn, char *vdata, size_t vn, unsigned int flags) { 53 | MDB_val key, val; 54 | LMDBGO_SET_VAL(&key, kn, kdata); 55 | LMDBGO_SET_VAL(&val, vn, vdata); 56 | return mdb_cursor_put(cur, &key, &val, flags); 57 | } 58 | 59 | int lmdbgo_mdb_cursor_put1(MDB_cursor *cur, char *kdata, size_t kn, MDB_val *val, unsigned int flags) { 60 | MDB_val key; 61 | LMDBGO_SET_VAL(&key, kn, kdata); 62 | return mdb_cursor_put(cur, &key, val, flags); 63 | } 64 | 65 | int lmdbgo_mdb_cursor_putmulti(MDB_cursor *cur, char *kdata, size_t kn, char *vdata, size_t vn, size_t vstride, unsigned int flags) { 66 | MDB_val key, val[2]; 67 | LMDBGO_SET_VAL(&key, kn, kdata); 68 | LMDBGO_SET_VAL(&(val[0]), vstride, vdata); 69 | LMDBGO_SET_VAL(&(val[1]), vn, 0); 70 | return mdb_cursor_put(cur, &key, &val[0], flags); 71 | } 72 | 73 | int lmdbgo_mdb_cursor_get1(MDB_cursor *cur, char *kdata, size_t kn, MDB_val *key, MDB_val *val, MDB_cursor_op op) { 74 | LMDBGO_SET_VAL(key, kn, kdata); 75 | return mdb_cursor_get(cur, key, val, op); 76 | } 77 | 78 | int lmdbgo_mdb_cursor_get2(MDB_cursor *cur, char *kdata, size_t kn, char *vdata, size_t vn, MDB_val *key, MDB_val *val, MDB_cursor_op op) { 79 | LMDBGO_SET_VAL(key, kn, kdata); 80 | LMDBGO_SET_VAL(val, vn, vdata); 81 | return mdb_cursor_get(cur, key, val, op); 82 | } 83 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/lmdbgo.h: -------------------------------------------------------------------------------- 1 | /* lmdbgo.h 2 | * Helper utilities for github.com/bmatsuo/lmdb-go/lmdb. These functions have 3 | * no compatibility guarantees and may be modified or deleted without warning. 4 | * */ 5 | #ifndef _LMDBGO_H_ 6 | #define _LMDBGO_H_ 7 | 8 | #include "lmdb.h" 9 | 10 | /* Proxy functions for lmdb get/put operations. The functions are defined to 11 | * take char* values instead of void* to keep cgo from cheking their data for 12 | * nested pointers and causing a couple of allocations per argument. 13 | * 14 | * For more information see github issues for more information about the 15 | * problem and the decision. 16 | * https://github.com/golang/go/issues/14387 17 | * https://github.com/golang/go/issues/15048 18 | * https://github.com/bmatsuo/lmdb-go/issues/63 19 | * */ 20 | int lmdbgo_mdb_del(MDB_txn *txn, MDB_dbi dbi, char *kdata, size_t kn, char *vdata, size_t vn); 21 | int lmdbgo_mdb_get(MDB_txn *txn, MDB_dbi dbi, char *kdata, size_t kn, MDB_val *val); 22 | int lmdbgo_mdb_put1(MDB_txn *txn, MDB_dbi dbi, char *kdata, size_t kn, MDB_val *val, unsigned int flags); 23 | int lmdbgo_mdb_put2(MDB_txn *txn, MDB_dbi dbi, char *kdata, size_t kn, char *vdata, size_t vn, unsigned int flags); 24 | int lmdbgo_mdb_cursor_put1(MDB_cursor *cur, char *kdata, size_t kn, MDB_val *val, unsigned int flags); 25 | int lmdbgo_mdb_cursor_put2(MDB_cursor *cur, char *kdata, size_t kn, char *vdata, size_t vn, unsigned int flags); 26 | int lmdbgo_mdb_cursor_putmulti(MDB_cursor *cur, char *kdata, size_t kn, char *vdata, size_t vn, size_t vstride, unsigned int flags); 27 | int lmdbgo_mdb_cursor_get1(MDB_cursor *cur, char *kdata, size_t kn, MDB_val *key, MDB_val *val, MDB_cursor_op op); 28 | int lmdbgo_mdb_cursor_get2(MDB_cursor *cur, char *kdata, size_t kn, char *vdata, size_t vn, MDB_val *key, MDB_val *val, MDB_cursor_op op); 29 | 30 | /* ConstCString wraps a null-terminated (const char *) because Go's type system 31 | * does not represent the 'cosnt' qualifier directly on a function argument and 32 | * causes warnings to be emitted during linking. 33 | * */ 34 | typedef struct{ const char *p; } lmdbgo_ConstCString; 35 | 36 | /* lmdbgo_mdb_reader_list is a proxy for mdb_reader_list that uses a special 37 | * mdb_msg_func proxy function to relay messages over the 38 | * lmdbgo_mdb_reader_list_bridge external Go func. 39 | * */ 40 | int lmdbgo_mdb_reader_list(MDB_env *env, size_t ctx); 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/midl.c: -------------------------------------------------------------------------------- 1 | /** @file midl.c 2 | * @brief ldap bdb back-end ID List functions */ 3 | /* $OpenLDAP$ */ 4 | /* This work is part of OpenLDAP Software . 5 | * 6 | * Copyright 2000-2015 The OpenLDAP Foundation. 7 | * All rights reserved. 8 | * 9 | * Redistribution and use in source and binary forms, with or without 10 | * modification, are permitted only as authorized by the OpenLDAP 11 | * Public License. 12 | * 13 | * A copy of this license is available in the file LICENSE in the 14 | * top-level directory of the distribution or, alternatively, at 15 | * . 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include "midl.h" 24 | 25 | /** @defgroup internal LMDB Internals 26 | * @{ 27 | */ 28 | /** @defgroup idls ID List Management 29 | * @{ 30 | */ 31 | #define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) 32 | 33 | unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) 34 | { 35 | /* 36 | * binary search of id in ids 37 | * if found, returns position of id 38 | * if not found, returns first position greater than id 39 | */ 40 | unsigned base = 0; 41 | unsigned cursor = 1; 42 | int val = 0; 43 | unsigned n = ids[0]; 44 | 45 | while( 0 < n ) { 46 | unsigned pivot = n >> 1; 47 | cursor = base + pivot + 1; 48 | val = CMP( ids[cursor], id ); 49 | 50 | if( val < 0 ) { 51 | n = pivot; 52 | 53 | } else if ( val > 0 ) { 54 | base = cursor; 55 | n -= pivot + 1; 56 | 57 | } else { 58 | return cursor; 59 | } 60 | } 61 | 62 | if( val > 0 ) { 63 | ++cursor; 64 | } 65 | return cursor; 66 | } 67 | 68 | #if 0 /* superseded by append/sort */ 69 | int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) 70 | { 71 | unsigned x, i; 72 | 73 | x = mdb_midl_search( ids, id ); 74 | assert( x > 0 ); 75 | 76 | if( x < 1 ) { 77 | /* internal error */ 78 | return -2; 79 | } 80 | 81 | if ( x <= ids[0] && ids[x] == id ) { 82 | /* duplicate */ 83 | assert(0); 84 | return -1; 85 | } 86 | 87 | if ( ++ids[0] >= MDB_IDL_DB_MAX ) { 88 | /* no room */ 89 | --ids[0]; 90 | return -2; 91 | 92 | } else { 93 | /* insert id */ 94 | for (i=ids[0]; i>x; i--) 95 | ids[i] = ids[i-1]; 96 | ids[x] = id; 97 | } 98 | 99 | return 0; 100 | } 101 | #endif 102 | 103 | MDB_IDL mdb_midl_alloc(int num) 104 | { 105 | MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID)); 106 | if (ids) { 107 | *ids++ = num; 108 | *ids = 0; 109 | } 110 | return ids; 111 | } 112 | 113 | void mdb_midl_free(MDB_IDL ids) 114 | { 115 | if (ids) 116 | free(ids-1); 117 | } 118 | 119 | void mdb_midl_shrink( MDB_IDL *idp ) 120 | { 121 | MDB_IDL ids = *idp; 122 | if (*(--ids) > MDB_IDL_UM_MAX && 123 | (ids = realloc(ids, (MDB_IDL_UM_MAX+2) * sizeof(MDB_ID)))) 124 | { 125 | *ids++ = MDB_IDL_UM_MAX; 126 | *idp = ids; 127 | } 128 | } 129 | 130 | static int mdb_midl_grow( MDB_IDL *idp, int num ) 131 | { 132 | MDB_IDL idn = *idp-1; 133 | /* grow it */ 134 | idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID)); 135 | if (!idn) 136 | return ENOMEM; 137 | *idn++ += num; 138 | *idp = idn; 139 | return 0; 140 | } 141 | 142 | int mdb_midl_need( MDB_IDL *idp, unsigned num ) 143 | { 144 | MDB_IDL ids = *idp; 145 | num += ids[0]; 146 | if (num > ids[-1]) { 147 | num = (num + num/4 + (256 + 2)) & -256; 148 | if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) 149 | return ENOMEM; 150 | *ids++ = num - 2; 151 | *idp = ids; 152 | } 153 | return 0; 154 | } 155 | 156 | int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) 157 | { 158 | MDB_IDL ids = *idp; 159 | /* Too big? */ 160 | if (ids[0] >= ids[-1]) { 161 | if (mdb_midl_grow(idp, MDB_IDL_UM_MAX)) 162 | return ENOMEM; 163 | ids = *idp; 164 | } 165 | ids[0]++; 166 | ids[ids[0]] = id; 167 | return 0; 168 | } 169 | 170 | int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) 171 | { 172 | MDB_IDL ids = *idp; 173 | /* Too big? */ 174 | if (ids[0] + app[0] >= ids[-1]) { 175 | if (mdb_midl_grow(idp, app[0])) 176 | return ENOMEM; 177 | ids = *idp; 178 | } 179 | memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); 180 | ids[0] += app[0]; 181 | return 0; 182 | } 183 | 184 | int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ) 185 | { 186 | MDB_ID *ids = *idp, len = ids[0]; 187 | /* Too big? */ 188 | if (len + n > ids[-1]) { 189 | if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX)) 190 | return ENOMEM; 191 | ids = *idp; 192 | } 193 | ids[0] = len + n; 194 | ids += len; 195 | while (n) 196 | ids[n--] = id++; 197 | return 0; 198 | } 199 | 200 | void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ) 201 | { 202 | MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k; 203 | idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */ 204 | old_id = idl[j]; 205 | while (i) { 206 | merge_id = merge[i--]; 207 | for (; old_id < merge_id; old_id = idl[--j]) 208 | idl[k--] = old_id; 209 | idl[k--] = merge_id; 210 | } 211 | idl[0] = total; 212 | } 213 | 214 | /* Quicksort + Insertion sort for small arrays */ 215 | 216 | #define SMALL 8 217 | #define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; } 218 | 219 | void 220 | mdb_midl_sort( MDB_IDL ids ) 221 | { 222 | /* Max possible depth of int-indexed tree * 2 items/level */ 223 | int istack[sizeof(int)*CHAR_BIT * 2]; 224 | int i,j,k,l,ir,jstack; 225 | MDB_ID a, itmp; 226 | 227 | ir = (int)ids[0]; 228 | l = 1; 229 | jstack = 0; 230 | for(;;) { 231 | if (ir - l < SMALL) { /* Insertion sort */ 232 | for (j=l+1;j<=ir;j++) { 233 | a = ids[j]; 234 | for (i=j-1;i>=1;i--) { 235 | if (ids[i] >= a) break; 236 | ids[i+1] = ids[i]; 237 | } 238 | ids[i+1] = a; 239 | } 240 | if (jstack == 0) break; 241 | ir = istack[jstack--]; 242 | l = istack[jstack--]; 243 | } else { 244 | k = (l + ir) >> 1; /* Choose median of left, center, right */ 245 | MIDL_SWAP(ids[k], ids[l+1]); 246 | if (ids[l] < ids[ir]) { 247 | MIDL_SWAP(ids[l], ids[ir]); 248 | } 249 | if (ids[l+1] < ids[ir]) { 250 | MIDL_SWAP(ids[l+1], ids[ir]); 251 | } 252 | if (ids[l] < ids[l+1]) { 253 | MIDL_SWAP(ids[l], ids[l+1]); 254 | } 255 | i = l+1; 256 | j = ir; 257 | a = ids[l+1]; 258 | for(;;) { 259 | do i++; while(ids[i] > a); 260 | do j--; while(ids[j] < a); 261 | if (j < i) break; 262 | MIDL_SWAP(ids[i],ids[j]); 263 | } 264 | ids[l+1] = ids[j]; 265 | ids[j] = a; 266 | jstack += 2; 267 | if (ir-i+1 >= j-l) { 268 | istack[jstack] = ir; 269 | istack[jstack-1] = i; 270 | ir = j-1; 271 | } else { 272 | istack[jstack] = j-1; 273 | istack[jstack-1] = l; 274 | l = i; 275 | } 276 | } 277 | } 278 | } 279 | 280 | unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ) 281 | { 282 | /* 283 | * binary search of id in ids 284 | * if found, returns position of id 285 | * if not found, returns first position greater than id 286 | */ 287 | unsigned base = 0; 288 | unsigned cursor = 1; 289 | int val = 0; 290 | unsigned n = (unsigned)ids[0].mid; 291 | 292 | while( 0 < n ) { 293 | unsigned pivot = n >> 1; 294 | cursor = base + pivot + 1; 295 | val = CMP( id, ids[cursor].mid ); 296 | 297 | if( val < 0 ) { 298 | n = pivot; 299 | 300 | } else if ( val > 0 ) { 301 | base = cursor; 302 | n -= pivot + 1; 303 | 304 | } else { 305 | return cursor; 306 | } 307 | } 308 | 309 | if( val > 0 ) { 310 | ++cursor; 311 | } 312 | return cursor; 313 | } 314 | 315 | int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ) 316 | { 317 | unsigned x, i; 318 | 319 | x = mdb_mid2l_search( ids, id->mid ); 320 | 321 | if( x < 1 ) { 322 | /* internal error */ 323 | return -2; 324 | } 325 | 326 | if ( x <= ids[0].mid && ids[x].mid == id->mid ) { 327 | /* duplicate */ 328 | return -1; 329 | } 330 | 331 | if ( ids[0].mid >= MDB_IDL_UM_MAX ) { 332 | /* too big */ 333 | return -2; 334 | 335 | } else { 336 | /* insert id */ 337 | ids[0].mid++; 338 | for (i=(unsigned)ids[0].mid; i>x; i--) 339 | ids[i] = ids[i-1]; 340 | ids[x] = *id; 341 | } 342 | 343 | return 0; 344 | } 345 | 346 | int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) 347 | { 348 | /* Too big? */ 349 | if (ids[0].mid >= MDB_IDL_UM_MAX) { 350 | return -2; 351 | } 352 | ids[0].mid++; 353 | ids[ids[0].mid] = *id; 354 | return 0; 355 | } 356 | 357 | /** @} */ 358 | /** @} */ 359 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/midl.h: -------------------------------------------------------------------------------- 1 | /** @file midl.h 2 | * @brief LMDB ID List header file. 3 | * 4 | * This file was originally part of back-bdb but has been 5 | * modified for use in libmdb. Most of the macros defined 6 | * in this file are unused, just left over from the original. 7 | * 8 | * This file is only used internally in libmdb and its definitions 9 | * are not exposed publicly. 10 | */ 11 | /* $OpenLDAP$ */ 12 | /* This work is part of OpenLDAP Software . 13 | * 14 | * Copyright 2000-2015 The OpenLDAP Foundation. 15 | * All rights reserved. 16 | * 17 | * Redistribution and use in source and binary forms, with or without 18 | * modification, are permitted only as authorized by the OpenLDAP 19 | * Public License. 20 | * 21 | * A copy of this license is available in the file LICENSE in the 22 | * top-level directory of the distribution or, alternatively, at 23 | * . 24 | */ 25 | 26 | #ifndef _MDB_MIDL_H_ 27 | #define _MDB_MIDL_H_ 28 | 29 | #include 30 | 31 | #ifdef __cplusplus 32 | extern "C" { 33 | #endif 34 | 35 | /** @defgroup internal LMDB Internals 36 | * @{ 37 | */ 38 | 39 | /** @defgroup idls ID List Management 40 | * @{ 41 | */ 42 | /** A generic unsigned ID number. These were entryIDs in back-bdb. 43 | * Preferably it should have the same size as a pointer. 44 | */ 45 | typedef size_t MDB_ID; 46 | 47 | /** An IDL is an ID List, a sorted array of IDs. The first 48 | * element of the array is a counter for how many actual 49 | * IDs are in the list. In the original back-bdb code, IDLs are 50 | * sorted in ascending order. For libmdb IDLs are sorted in 51 | * descending order. 52 | */ 53 | typedef MDB_ID *MDB_IDL; 54 | 55 | /* IDL sizes - likely should be even bigger 56 | * limiting factors: sizeof(ID), thread stack size 57 | */ 58 | #define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ 59 | #define MDB_IDL_DB_SIZE (1< 5 | #include 6 | #include "lmdb.h" 7 | #include "lmdbgo.h" 8 | */ 9 | import "C" 10 | 11 | import ( 12 | "log" 13 | "runtime" 14 | "unsafe" 15 | ) 16 | 17 | // This flags are used exclusively for Txn.OpenDBI and Txn.OpenRoot. The 18 | // Create flag must always be supplied when opening a non-root DBI for the 19 | // first time. 20 | // 21 | // BUG(bmatsuo): 22 | // MDB_INTEGERKEY and MDB_INTEGERDUP aren't usable. I'm not sure they would be 23 | // faster with the cgo bridge. They need to be tested and benchmarked. 24 | const ( 25 | // Flags for Txn.OpenDBI. 26 | 27 | ReverseKey = C.MDB_REVERSEKEY // Use reverse string keys. 28 | DupSort = C.MDB_DUPSORT // Use sorted duplicates. 29 | DupFixed = C.MDB_DUPFIXED // Duplicate items have a fixed size (DupSort). 30 | ReverseDup = C.MDB_REVERSEDUP // Reverse duplicate values (DupSort). 31 | Create = C.MDB_CREATE // Create DB if not already existing. 32 | ) 33 | 34 | // Txn is a database transaction in an environment. 35 | // 36 | // WARNING: A writable Txn is not threadsafe and may only be used in the 37 | // goroutine that created it. 38 | // 39 | // See MDB_txn. 40 | type Txn struct { 41 | // If RawRead is true []byte values retrieved from Get() calls on the Txn 42 | // and its cursors will point directly into the memory-mapped structure. 43 | // Such slices will be readonly and must only be referenced wthin the 44 | // transaction's lifetime. 45 | RawRead bool 46 | managed bool 47 | readonly bool 48 | env *Env 49 | _txn *C.MDB_txn 50 | errLogf func(format string, v ...interface{}) 51 | key *C.MDB_val 52 | val *C.MDB_val 53 | } 54 | 55 | // beginTxn does not lock the OS thread which is a prerequisite for creating a 56 | // write transaction. 57 | func beginTxn(env *Env, parent *Txn, flags uint) (*Txn, error) { 58 | txn := &Txn{ 59 | readonly: (flags&Readonly != 0), 60 | env: env, 61 | } 62 | 63 | var ptxn *C.MDB_txn 64 | if parent == nil { 65 | ptxn = nil 66 | txn.key = new(C.MDB_val) 67 | txn.val = new(C.MDB_val) 68 | } else { 69 | ptxn = parent._txn 70 | txn.key = parent.key 71 | txn.val = parent.val 72 | } 73 | ret := C.mdb_txn_begin(env._env, ptxn, C.uint(flags), &txn._txn) 74 | if ret != success { 75 | return nil, operrno("mdb_txn_begin", ret) 76 | } 77 | return txn, nil 78 | } 79 | 80 | // ID returns the identifier for txn. A view transaction identifier 81 | // corresponds to the Env snapshot being viewed and may be shared with other 82 | // view transactions. 83 | // 84 | // See mdb_txn_id. 85 | func (txn *Txn) ID() uintptr { 86 | return uintptr(C.mdb_txn_id(txn._txn)) 87 | } 88 | 89 | // Commit persists all transaction operations to the database and clears the 90 | // finalizer on txn. A Txn cannot be used again after Commit is called. 91 | // 92 | // See mdb_txn_commit. 93 | func (txn *Txn) Commit() error { 94 | if txn.managed { 95 | panic("managed transaction cannot be comitted directly") 96 | } 97 | runtime.SetFinalizer(txn, nil) 98 | return txn.commit() 99 | } 100 | 101 | func (txn *Txn) commit() error { 102 | ret := C.mdb_txn_commit(txn._txn) 103 | txn._txn = nil 104 | return operrno("mdb_txn_commit", ret) 105 | } 106 | 107 | // Abort discards pending writes in the transaction and clears the finalizer on 108 | // txn. A Txn cannot be used again after Abort is called. 109 | // 110 | // See mdb_txn_abort. 111 | func (txn *Txn) Abort() { 112 | if txn.managed { 113 | panic("managed transaction cannot be aborted directly") 114 | } 115 | runtime.SetFinalizer(txn, nil) 116 | txn.abort() 117 | } 118 | 119 | func (txn *Txn) abort() { 120 | if txn._txn == nil { 121 | return 122 | } 123 | C.mdb_txn_abort(txn._txn) 124 | // The transaction handle is always freed. 125 | txn._txn = nil 126 | } 127 | 128 | // Reset aborts the transaction clears internal state so the transaction may be 129 | // reused by calling Renew. If txn is not going to be reused txn.Abort() must 130 | // be called to release its slot in the lock table and free its memory. Reset 131 | // panics if txn is managed by Update, View, etc. 132 | // 133 | // See mdb_txn_reset. 134 | func (txn *Txn) Reset() { 135 | if txn.managed { 136 | panic("managed transaction cannot be reset directly") 137 | } 138 | txn.reset() 139 | runtime.SetFinalizer(txn, nil) 140 | } 141 | 142 | func (txn *Txn) reset() { 143 | C.mdb_txn_reset(txn._txn) 144 | } 145 | 146 | // Renew reuses a transaction that was previously reset by calling txn.Reset(). 147 | // Renew panics if txn is managed by Update, View, etc. 148 | // 149 | // See mdb_txn_renew. 150 | func (txn *Txn) Renew() error { 151 | if txn.managed { 152 | panic("managed transaction cannot be renewed directly") 153 | } 154 | err := txn.renew() 155 | if err != nil { 156 | return err 157 | } 158 | runtime.SetFinalizer(txn, func(v interface{}) { v.(*Txn).finalize() }) 159 | return nil 160 | } 161 | 162 | func (txn *Txn) renew() error { 163 | ret := C.mdb_txn_renew(txn._txn) 164 | return operrno("mdb_txn_renew", ret) 165 | } 166 | 167 | // OpenDBI opens a named database in the environment. An error is returned if 168 | // name is empty. The DBI returned by OpenDBI can be used in other 169 | // transactions but not before Txn has terminated. 170 | // 171 | // OpenDBI can only be called after env.SetMaxDBs() has been called to set the 172 | // maximum number of named databases. 173 | // 174 | // The C API uses null terminated strings for database names. A consequence is 175 | // that names cannot contain null bytes themselves. OpenDBI does not check for 176 | // null bytes in the name argument. 177 | // 178 | // See mdb_dbi_open. 179 | func (txn *Txn) OpenDBI(name string, flags uint) (DBI, error) { 180 | cname := C.CString(name) 181 | dbi, err := txn.openDBI(cname, flags) 182 | C.free(unsafe.Pointer(cname)) 183 | return dbi, err 184 | } 185 | 186 | // CreateDBI is a shorthand for OpenDBI that passed the flag lmdb.Create. 187 | func (txn *Txn) CreateDBI(name string) (DBI, error) { 188 | return txn.OpenDBI(name, Create) 189 | } 190 | 191 | // Flags returns the database flags for handle dbi. 192 | func (txn *Txn) Flags(dbi DBI) (uint, error) { 193 | var cflags C.uint 194 | ret := C.mdb_dbi_flags(txn._txn, C.MDB_dbi(dbi), (*C.uint)(&cflags)) 195 | return uint(cflags), operrno("mdb_dbi_flags", ret) 196 | } 197 | 198 | // OpenRoot opens the root database. OpenRoot behaves similarly to OpenDBI but 199 | // does not require env.SetMaxDBs() to be called beforehand. And, OpenRoot can 200 | // be called without flags in a View transaction. 201 | func (txn *Txn) OpenRoot(flags uint) (DBI, error) { 202 | return txn.openDBI(nil, flags) 203 | } 204 | 205 | // openDBI returns returns whatever DBI value was set by mdb_open_dbi. In an 206 | // error case, LMDB does not currently set DBI in case of failure, so zero is 207 | // returned in those cases. This is not a big deal for now because 208 | // applications are expected to handle any error encountered opening a 209 | // database. 210 | func (txn *Txn) openDBI(cname *C.char, flags uint) (DBI, error) { 211 | var dbi C.MDB_dbi 212 | ret := C.mdb_dbi_open(txn._txn, cname, C.uint(flags), &dbi) 213 | return DBI(dbi), operrno("mdb_dbi_open", ret) 214 | } 215 | 216 | // Stat returns a Stat describing the database dbi. 217 | // 218 | // See mdb_stat. 219 | func (txn *Txn) Stat(dbi DBI) (*Stat, error) { 220 | var _stat C.MDB_stat 221 | ret := C.mdb_stat(txn._txn, C.MDB_dbi(dbi), &_stat) 222 | if ret != success { 223 | return nil, operrno("mdb_stat", ret) 224 | } 225 | stat := Stat{PSize: uint(_stat.ms_psize), 226 | Depth: uint(_stat.ms_depth), 227 | BranchPages: uint64(_stat.ms_branch_pages), 228 | LeafPages: uint64(_stat.ms_leaf_pages), 229 | OverflowPages: uint64(_stat.ms_overflow_pages), 230 | Entries: uint64(_stat.ms_entries)} 231 | return &stat, nil 232 | } 233 | 234 | // Drop empties the database if del is false. Drop deletes and closes the 235 | // database if del is true. 236 | // 237 | // See mdb_drop. 238 | func (txn *Txn) Drop(dbi DBI, del bool) error { 239 | ret := C.mdb_drop(txn._txn, C.MDB_dbi(dbi), cbool(del)) 240 | return operrno("mdb_drop", ret) 241 | } 242 | 243 | // Sub executes fn in a subtransaction. Sub commits the subtransaction iff a 244 | // nil error is returned by fn and otherwise aborts it. Sub returns any error 245 | // it encounters. 246 | // 247 | // Sub may only be called on an Update (a Txn created without the Readonly 248 | // flag). Calling Sub on a View transaction will return an error. 249 | // 250 | // Any call to Abort, Commit, Renew, or Reset on a Txn created by Sub will 251 | // panic. 252 | func (txn *Txn) Sub(fn TxnOp) error { 253 | // As of 0.9.14 Readonly is the only Txn flag and readonly subtransactions 254 | // don't make sense. 255 | return txn.subFlag(0, fn) 256 | } 257 | 258 | func (txn *Txn) subFlag(flags uint, fn TxnOp) error { 259 | sub, err := beginTxn(txn.env, txn, flags) 260 | if err != nil { 261 | return err 262 | } 263 | sub.managed = true 264 | defer sub.abort() 265 | err = fn(sub) 266 | if err != nil { 267 | return err 268 | } 269 | return sub.commit() 270 | } 271 | 272 | func (txn *Txn) bytes(val *C.MDB_val) []byte { 273 | if txn.RawRead { 274 | return getBytes(val) 275 | } 276 | return getBytesCopy(val) 277 | } 278 | 279 | // Get retrieves items from database dbi. If txn.RawRead is true the slice 280 | // returned by Get references a readonly section of memory that must not be 281 | // accessed after txn has terminated. 282 | // 283 | // See mdb_get. 284 | func (txn *Txn) Get(dbi DBI, key []byte) ([]byte, error) { 285 | kdata, kn := valBytes(key) 286 | ret := C.lmdbgo_mdb_get( 287 | txn._txn, C.MDB_dbi(dbi), 288 | (*C.char)(unsafe.Pointer(&kdata[0])), C.size_t(kn), 289 | txn.val, 290 | ) 291 | err := operrno("mdb_get", ret) 292 | if err != nil { 293 | *txn.val = C.MDB_val{} 294 | return nil, err 295 | } 296 | b := txn.bytes(txn.val) 297 | *txn.val = C.MDB_val{} 298 | return b, nil 299 | } 300 | 301 | func (txn *Txn) putNilKey(dbi DBI, flags uint) error { 302 | // mdb_put with an empty key will always fail 303 | ret := C.lmdbgo_mdb_put2(txn._txn, C.MDB_dbi(dbi), nil, 0, nil, 0, C.uint(flags)) 304 | return operrno("mdb_put", ret) 305 | } 306 | 307 | // Put stores an item in database dbi. 308 | // 309 | // See mdb_put. 310 | func (txn *Txn) Put(dbi DBI, key []byte, val []byte, flags uint) error { 311 | kn := len(key) 312 | if kn == 0 { 313 | return txn.putNilKey(dbi, flags) 314 | } 315 | vn := len(val) 316 | if vn == 0 { 317 | val = []byte{0} 318 | } 319 | 320 | ret := C.lmdbgo_mdb_put2( 321 | txn._txn, C.MDB_dbi(dbi), 322 | (*C.char)(unsafe.Pointer(&key[0])), C.size_t(kn), 323 | (*C.char)(unsafe.Pointer(&val[0])), C.size_t(vn), 324 | C.uint(flags), 325 | ) 326 | return operrno("mdb_put", ret) 327 | } 328 | 329 | // PutReserve returns a []byte of length n that can be written to, potentially 330 | // avoiding a memcopy. The returned byte slice is only valid in txn's thread, 331 | // before it has terminated. 332 | func (txn *Txn) PutReserve(dbi DBI, key []byte, n int, flags uint) ([]byte, error) { 333 | if len(key) == 0 { 334 | return nil, txn.putNilKey(dbi, flags) 335 | } 336 | txn.val.mv_size = C.size_t(n) 337 | ret := C.lmdbgo_mdb_put1( 338 | txn._txn, C.MDB_dbi(dbi), 339 | (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key)), 340 | txn.val, 341 | C.uint(flags|C.MDB_RESERVE), 342 | ) 343 | err := operrno("mdb_put", ret) 344 | if err != nil { 345 | *txn.val = C.MDB_val{} 346 | return nil, err 347 | } 348 | b := getBytes(txn.val) 349 | *txn.val = C.MDB_val{} 350 | return b, nil 351 | } 352 | 353 | // Del deletes an item from database dbi. Del ignores val unless dbi has the 354 | // DupSort flag. 355 | // 356 | // See mdb_del. 357 | func (txn *Txn) Del(dbi DBI, key, val []byte) error { 358 | kdata, kn := valBytes(key) 359 | vdata, vn := valBytes(val) 360 | ret := C.lmdbgo_mdb_del( 361 | txn._txn, C.MDB_dbi(dbi), 362 | (*C.char)(unsafe.Pointer(&kdata[0])), C.size_t(kn), 363 | (*C.char)(unsafe.Pointer(&vdata[0])), C.size_t(vn), 364 | ) 365 | return operrno("mdb_del", ret) 366 | } 367 | 368 | // OpenCursor allocates and initializes a Cursor to database dbi. 369 | // 370 | // See mdb_cursor_open. 371 | func (txn *Txn) OpenCursor(dbi DBI) (*Cursor, error) { 372 | cur, err := openCursor(txn, dbi) 373 | if cur != nil && txn.readonly { 374 | runtime.SetFinalizer(cur, (*Cursor).close) 375 | } 376 | return cur, err 377 | } 378 | 379 | func (txn *Txn) errf(format string, v ...interface{}) { 380 | if txn.errLogf != nil { 381 | txn.errLogf(format, v...) 382 | return 383 | } 384 | log.Printf(format, v...) 385 | } 386 | 387 | func (txn *Txn) finalize() { 388 | if txn._txn != nil { 389 | txn.errf("lmdb: aborting unreachable transaction %#x", uintptr(unsafe.Pointer(txn))) 390 | txn.Abort() 391 | } 392 | } 393 | 394 | // TxnOp is an operation applied to a managed transaction. The Txn passed to a 395 | // TxnOp is managed and the operation must not call Commit, Abort, Renew, or 396 | // Reset on it. 397 | // 398 | // IMPORTANT: 399 | // TxnOps that write to the database (those passed to Env.Update or Txn.Sub) 400 | // must not use the Txn in another goroutine (passing it directly or otherwise 401 | // through closure). Doing so has undefined results. 402 | type TxnOp func(txn *Txn) error 403 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/val.go: -------------------------------------------------------------------------------- 1 | package lmdb 2 | 3 | /* 4 | #include 5 | #include 6 | #include "lmdb.h" 7 | */ 8 | import "C" 9 | 10 | import "unsafe" 11 | 12 | // valMaxSize is the largest portable data size allowed by Go (larger can cause 13 | // an error like "type [...]byte larger than address space"). See runtime 14 | // source file malloc.go for more information about memory limits. 15 | // 16 | // https://github.com/golang/go/blob/a03bdc3e6bea34abd5077205371e6fb9ef354481/src/runtime/malloc.go#L151-L164 17 | // 18 | // Luckily, the value 2^32-1 coincides with the maximum data size for LMDB 19 | // (MAXDATASIZE). 20 | const valMaxSize = 1<<32 - 1 21 | 22 | // Multi is a wrapper for a contiguous page of sorted, fixed-length values 23 | // passed to Cursor.PutMulti or retrieved using Cursor.Get with the 24 | // GetMultiple/NextMultiple flag. 25 | // 26 | // Multi values are only useful in databases opened with DupSort|DupFixed. 27 | type Multi struct { 28 | page []byte 29 | stride int 30 | } 31 | 32 | // WrapMulti converts a page of contiguous values with stride size into a 33 | // Multi. WrapMulti panics if len(page) is not a multiple of stride. 34 | // 35 | // _, val, _ := cursor.Get(nil, nil, lmdb.FirstDup) 36 | // _, page, _ := cursor.Get(nil, nil, lmdb.GetMultiple) 37 | // multi := lmdb.WrapMulti(page, len(val)) 38 | // 39 | // See mdb_cursor_get and MDB_GET_MULTIPLE. 40 | func WrapMulti(page []byte, stride int) *Multi { 41 | if len(page)%stride != 0 { 42 | panic("incongruent arguments") 43 | } 44 | return &Multi{page: page, stride: stride} 45 | } 46 | 47 | // Vals returns a slice containing the values in m. The returned slice has 48 | // length m.Len() and each item has length m.Stride(). 49 | func (m *Multi) Vals() [][]byte { 50 | n := m.Len() 51 | ps := make([][]byte, n) 52 | for i := 0; i < n; i++ { 53 | ps[i] = m.Val(i) 54 | } 55 | return ps 56 | } 57 | 58 | // Val returns the value at index i. Val panics if i is out of range. 59 | func (m *Multi) Val(i int) []byte { 60 | off := i * m.stride 61 | return m.page[off : off+m.stride] 62 | } 63 | 64 | // Len returns the number of values in the Multi. 65 | func (m *Multi) Len() int { 66 | return len(m.page) / m.stride 67 | } 68 | 69 | // Stride returns the length of an individual value in the m. 70 | func (m *Multi) Stride() int { 71 | return m.stride 72 | } 73 | 74 | // Size returns the total size of the Multi data and is equal to 75 | // 76 | // m.Len()*m.Stride() 77 | // 78 | func (m *Multi) Size() int { 79 | return len(m.page) 80 | } 81 | 82 | // Page returns the Multi page data as a raw slice of bytes with length 83 | // m.Size(). 84 | func (m *Multi) Page() []byte { 85 | return m.page[:len(m.page):len(m.page)] 86 | } 87 | 88 | var eb = []byte{0} 89 | 90 | func valBytes(b []byte) ([]byte, int) { 91 | if len(b) == 0 { 92 | return eb, 0 93 | } 94 | return b, len(b) 95 | } 96 | 97 | func wrapVal(b []byte) *C.MDB_val { 98 | p, n := valBytes(b) 99 | return &C.MDB_val{ 100 | mv_data: unsafe.Pointer(&p[0]), 101 | mv_size: C.size_t(n), 102 | } 103 | } 104 | 105 | func getBytes(val *C.MDB_val) []byte { 106 | return (*[valMaxSize]byte)(unsafe.Pointer(val.mv_data))[:val.mv_size:val.mv_size] 107 | } 108 | 109 | func getBytesCopy(val *C.MDB_val) []byte { 110 | return C.GoBytes(val.mv_data, C.int(val.mv_size)) 111 | } 112 | -------------------------------------------------------------------------------- /vendor/github.com/bmatsuo/lmdb-go/lmdb/val_test.go: -------------------------------------------------------------------------------- 1 | package lmdb 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func TestMultiVal(t *testing.T) { 10 | data := []byte("abcdef") 11 | m := WrapMulti(data, 2) 12 | vals := m.Vals() 13 | if !reflect.DeepEqual(vals, [][]byte{{'a', 'b'}, {'c', 'd'}, {'e', 'f'}}) { 14 | t.Errorf("unexpected vals: %q", vals) 15 | } 16 | size := m.Size() 17 | if size != 6 { 18 | t.Errorf("unexpected size: %v (!= %v)", size, 6) 19 | } 20 | length := m.Len() 21 | if length != 3 { 22 | t.Errorf("unexpected length: %v (!= %v)", length, 3) 23 | } 24 | stride := m.Stride() 25 | if stride != 2 { 26 | t.Errorf("unexpected stride: %v (!= %v)", stride, 2) 27 | } 28 | page := m.Page() 29 | if !bytes.Equal(page, data) { 30 | t.Errorf("unexpected page: %v (!= %v)", page, data) 31 | } 32 | } 33 | 34 | func TestMultiVal_panic(t *testing.T) { 35 | var p bool 36 | defer func() { 37 | if e := recover(); e != nil { 38 | p = true 39 | } 40 | if !p { 41 | t.Errorf("expected a panic") 42 | } 43 | }() 44 | WrapMulti([]byte("123"), 2) 45 | } 46 | 47 | func TestValBytes(t *testing.T) { 48 | ptr, n := valBytes(nil) 49 | if len(ptr) == 0 { 50 | t.Errorf("unexpected unaddressable slice") 51 | } 52 | if n != 0 { 53 | t.Errorf("unexpected length: %d (expected 0)", n) 54 | } 55 | 56 | b := []byte("abc") 57 | ptr, n = valBytes(b) 58 | if len(ptr) == 0 { 59 | t.Errorf("unexpected unaddressable slice") 60 | } 61 | if n != 3 { 62 | t.Errorf("unexpected length: %d (expected %d)", n, len(b)) 63 | } 64 | } 65 | 66 | func TestVal(t *testing.T) { 67 | orig := []byte("hey hey") 68 | val := wrapVal(orig) 69 | 70 | p := getBytes(val) 71 | if !bytes.Equal(p, orig) { 72 | t.Errorf("getBytes() not the same as original data: %q", p) 73 | } 74 | if &p[0] != &orig[0] { 75 | t.Errorf("getBytes() is not the same slice as original") 76 | } 77 | 78 | p = getBytesCopy(val) 79 | if !bytes.Equal(p, orig) { 80 | t.Errorf("getBytesCopy() not the same as original data: %q", p) 81 | } 82 | if &p[0] == &orig[0] { 83 | t.Errorf("getBytesCopy() overlaps with orignal slice") 84 | } 85 | } 86 | --------------------------------------------------------------------------------