├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── README.md ├── client.go ├── client_test.go ├── config.go ├── config_test.go ├── doc.go ├── docker-compose.yml ├── errors.go ├── go.mod ├── go.sum ├── leveldbstore └── store.go ├── memorystore └── store.go ├── store.go ├── upload.go ├── upload_test.go ├── uploader.go └── vendor ├── github.com ├── bmizerany │ └── pat │ │ ├── .gitignore │ │ ├── README.md │ │ └── mux.go ├── davecgh │ └── go-spew │ │ ├── LICENSE │ │ └── spew │ │ ├── bypass.go │ │ ├── bypasssafe.go │ │ ├── common.go │ │ ├── config.go │ │ ├── doc.go │ │ ├── dump.go │ │ ├── format.go │ │ └── spew.go ├── golang │ └── snappy │ │ ├── .gitignore │ │ ├── AUTHORS │ │ ├── CONTRIBUTORS │ │ ├── LICENSE │ │ ├── README │ │ ├── decode.go │ │ ├── decode_amd64.go │ │ ├── decode_amd64.s │ │ ├── decode_other.go │ │ ├── encode.go │ │ ├── encode_amd64.go │ │ ├── encode_amd64.s │ │ ├── encode_other.go │ │ └── snappy.go ├── pmezard │ └── go-difflib │ │ ├── LICENSE │ │ └── difflib │ │ └── difflib.go ├── stretchr │ └── testify │ │ ├── LICENSE │ │ ├── assert │ │ ├── assertion_format.go │ │ ├── assertion_format.go.tmpl │ │ ├── assertion_forward.go │ │ ├── assertion_forward.go.tmpl │ │ ├── assertion_order.go │ │ ├── assertions.go │ │ ├── doc.go │ │ ├── errors.go │ │ ├── forward_assertions.go │ │ └── http_assertions.go │ │ ├── require │ │ ├── doc.go │ │ ├── forward_requirements.go │ │ ├── require.go │ │ ├── require.go.tmpl │ │ ├── require_forward.go │ │ ├── require_forward.go.tmpl │ │ └── requirements.go │ │ └── suite │ │ ├── doc.go │ │ ├── interfaces.go │ │ └── suite.go ├── syndtr │ └── goleveldb │ │ ├── LICENSE │ │ └── leveldb │ │ ├── batch.go │ │ ├── cache │ │ ├── cache.go │ │ └── lru.go │ │ ├── comparer.go │ │ ├── comparer │ │ ├── bytes_comparer.go │ │ └── comparer.go │ │ ├── db.go │ │ ├── db_compaction.go │ │ ├── db_iter.go │ │ ├── db_snapshot.go │ │ ├── db_state.go │ │ ├── db_transaction.go │ │ ├── db_util.go │ │ ├── db_write.go │ │ ├── doc.go │ │ ├── errors.go │ │ ├── errors │ │ └── errors.go │ │ ├── filter.go │ │ ├── filter │ │ ├── bloom.go │ │ └── filter.go │ │ ├── iterator │ │ ├── array_iter.go │ │ ├── indexed_iter.go │ │ ├── iter.go │ │ └── merged_iter.go │ │ ├── journal │ │ └── journal.go │ │ ├── key.go │ │ ├── memdb │ │ └── memdb.go │ │ ├── opt │ │ └── options.go │ │ ├── options.go │ │ ├── session.go │ │ ├── session_compaction.go │ │ ├── session_record.go │ │ ├── session_util.go │ │ ├── storage.go │ │ ├── storage │ │ ├── file_storage.go │ │ ├── file_storage_nacl.go │ │ ├── file_storage_plan9.go │ │ ├── file_storage_solaris.go │ │ ├── file_storage_unix.go │ │ ├── file_storage_windows.go │ │ ├── mem_storage.go │ │ └── storage.go │ │ ├── table.go │ │ ├── table │ │ ├── reader.go │ │ ├── table.go │ │ └── writer.go │ │ ├── util.go │ │ ├── util │ │ ├── buffer.go │ │ ├── buffer_pool.go │ │ ├── crc32.go │ │ ├── hash.go │ │ ├── range.go │ │ └── util.go │ │ └── version.go └── tus │ └── tusd │ ├── LICENSE.txt │ ├── internal │ └── uid │ │ └── uid.go │ └── pkg │ ├── filestore │ └── filestore.go │ └── handler │ ├── composer.go │ ├── composer.mgo │ ├── config.go │ ├── datastore.go │ ├── doc.go │ ├── handler.go │ ├── log.go │ ├── metrics.go │ └── unrouted_handler.go ├── gopkg.in └── yaml.v2 │ ├── .travis.yml │ ├── LICENSE │ ├── LICENSE.libyaml │ ├── NOTICE │ ├── README.md │ ├── apic.go │ ├── decode.go │ ├── emitterc.go │ ├── encode.go │ ├── go.mod │ ├── parserc.go │ ├── readerc.go │ ├── resolve.go │ ├── scannerc.go │ ├── sorter.go │ ├── writerc.go │ ├── yaml.go │ ├── yamlh.go │ └── yamlprivateh.go └── modules.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | .DS_Store 27 | 28 | .idea/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.13 4 | - tip 5 | script: go test -v ./... 6 | notifications: 7 | email: 8 | recipients: 9 | - devs@eventials.com 10 | on_success: never 11 | on_failure: always 12 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.13 2 | 3 | RUN mkdir -p /go/src/github.com/eventials/go-tus 4 | 5 | WORKDIR /go/src/github.com/eventials/go-tus 6 | 7 | RUN go get github.com/stretchr/testify 8 | RUN go get github.com/tus/tusd 9 | RUN go get github.com/syndtr/goleveldb/leveldb 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Eventials 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-tus [![Build Status](https://travis-ci.org/eventials/go-tus.svg?branch=master)](https://travis-ci.org/eventials/go-tus) [![Go Report Card](https://goreportcard.com/badge/github.com/eventials/go-tus)](https://goreportcard.com/report/github.com/eventials/go-tus) [![GoDoc](https://godoc.org/github.com/eventials/go-tus?status.svg)](http://godoc.org/github.com/eventials/go-tus) 2 | 3 | A pure Go client for the [tus resumable upload protocol](http://tus.io/) 4 | 5 | ## Example 6 | 7 | ```go 8 | package main 9 | 10 | import ( 11 | "os" 12 | "github.com/eventials/go-tus" 13 | ) 14 | 15 | func main() { 16 | f, err := os.Open("my-file.txt") 17 | 18 | if err != nil { 19 | panic(err) 20 | } 21 | 22 | defer f.Close() 23 | 24 | // create the tus client. 25 | client, _ := tus.NewClient("https://tus.example.org/files", nil) 26 | 27 | // create an upload from a file. 28 | upload, _ := tus.NewUploadFromFile(f) 29 | 30 | // create the uploader. 31 | uploader, _ := client.CreateUpload(upload) 32 | 33 | // start the uploading process. 34 | uploader.Upload() 35 | } 36 | ``` 37 | 38 | ## Features 39 | 40 | > This is not a full protocol client implementation. 41 | 42 | Checksum, Termination and Concatenation extensions are not implemented yet. 43 | 44 | This client allows to resume an upload if a Store is used. 45 | 46 | ## Built in Store 47 | 48 | Store is used to map an upload's fingerprint with the corresponding upload URL. 49 | 50 | | Name | Backend | Dependencies | 51 | |:----:|:-------:|:------------:| 52 | | MemoryStore | In-Memory | None | 53 | | LeveldbStore | LevelDB | [goleveldb](https://github.com/syndtr/goleveldb) | 54 | 55 | ## Future Work 56 | 57 | - [ ] SQLite store 58 | - [ ] Redis store 59 | - [ ] Memcached store 60 | - [ ] Checksum extension 61 | - [ ] Termination extension 62 | - [ ] Concatenation extension 63 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package tus 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "net/http" 8 | netUrl "net/url" 9 | "strconv" 10 | ) 11 | 12 | const ( 13 | ProtocolVersion = "1.0.0" 14 | ) 15 | 16 | // Client represents the tus client. 17 | // You can use it in goroutines to create parallels uploads. 18 | type Client struct { 19 | Config *Config 20 | Url string 21 | Version string 22 | Header http.Header 23 | 24 | client *http.Client 25 | } 26 | 27 | // NewClient creates a new tus client. 28 | func NewClient(url string, config *Config) (*Client, error) { 29 | if config == nil { 30 | config = DefaultConfig() 31 | } else { 32 | if err := config.Validate(); err != nil { 33 | return nil, err 34 | } 35 | } 36 | 37 | if config.Header == nil { 38 | config.Header = make(http.Header) 39 | } 40 | 41 | if config.HttpClient == nil { 42 | config.HttpClient = &http.Client{} 43 | } 44 | 45 | return &Client{ 46 | Config: config, 47 | Url: url, 48 | Version: ProtocolVersion, 49 | Header: config.Header, 50 | 51 | client: config.HttpClient, 52 | }, nil 53 | } 54 | 55 | func (c *Client) Do(req *http.Request) (*http.Response, error) { 56 | for k, v := range c.Header { 57 | req.Header[k] = v 58 | } 59 | 60 | req.Header.Set("Tus-Resumable", ProtocolVersion) 61 | 62 | return c.client.Do(req) 63 | } 64 | 65 | // CreateUpload creates a new upload in the server. 66 | func (c *Client) CreateUpload(u *Upload) (*Uploader, error) { 67 | if u == nil { 68 | return nil, ErrNilUpload 69 | } 70 | 71 | if c.Config.Resume && len(u.Fingerprint) == 0 { 72 | return nil, ErrFingerprintNotSet 73 | } 74 | 75 | req, err := http.NewRequest("POST", c.Url, nil) 76 | 77 | if err != nil { 78 | return nil, err 79 | } 80 | 81 | req.Header.Set("Content-Length", "0") 82 | req.Header.Set("Upload-Length", strconv.FormatInt(u.size, 10)) 83 | req.Header.Set("Upload-Metadata", u.EncodedMetadata()) 84 | 85 | res, err := c.Do(req) 86 | 87 | if err != nil { 88 | return nil, err 89 | } 90 | defer res.Body.Close() 91 | 92 | switch res.StatusCode { 93 | case 201: 94 | location := res.Header.Get("Location") 95 | 96 | newURL, err := c.resolveLocationURL(location) 97 | if err != nil { 98 | return nil, err 99 | } 100 | 101 | if c.Config.Resume { 102 | c.Config.Store.Set(u.Fingerprint, newURL.String()) 103 | } 104 | 105 | return NewUploader(c, newURL.String(), u, 0), nil 106 | case 412: 107 | return nil, ErrVersionMismatch 108 | case 413: 109 | return nil, ErrLargeUpload 110 | default: 111 | return nil, newClientError(res) 112 | } 113 | } 114 | 115 | func (c *Client) resolveLocationURL(location string) (*netUrl.URL, error) { 116 | baseURL, err := netUrl.Parse(c.Url) 117 | if err != nil { 118 | return nil, fmt.Errorf("Invalid URL '%s'", c.Url) 119 | } 120 | 121 | locationURL, err := netUrl.Parse(location) 122 | if err != nil { 123 | return nil, fmt.Errorf("Invalid Location header value for a Creation '%s': %s", location, err.Error()) 124 | } 125 | newURL := baseURL.ResolveReference(locationURL) 126 | return newURL, nil 127 | } 128 | 129 | // ResumeUpload resumes the upload if already created, otherwise it will return an error. 130 | func (c *Client) ResumeUpload(u *Upload) (*Uploader, error) { 131 | if u == nil { 132 | return nil, ErrNilUpload 133 | } 134 | 135 | if !c.Config.Resume { 136 | return nil, ErrResumeNotEnabled 137 | } else if len(u.Fingerprint) == 0 { 138 | return nil, ErrFingerprintNotSet 139 | } 140 | 141 | url, found := c.Config.Store.Get(u.Fingerprint) 142 | 143 | if !found { 144 | return nil, ErrUploadNotFound 145 | } 146 | 147 | offset, err := c.getUploadOffset(url) 148 | 149 | if err != nil { 150 | return nil, err 151 | } 152 | 153 | return NewUploader(c, url, u, offset), nil 154 | } 155 | 156 | // CreateOrResumeUpload resumes the upload if already created or creates a new upload in the server. 157 | func (c *Client) CreateOrResumeUpload(u *Upload) (*Uploader, error) { 158 | if u == nil { 159 | return nil, ErrNilUpload 160 | } 161 | 162 | uploader, err := c.ResumeUpload(u) 163 | 164 | if err == nil { 165 | return uploader, err 166 | } else if (err == ErrResumeNotEnabled) || (err == ErrUploadNotFound) { 167 | return c.CreateUpload(u) 168 | } 169 | 170 | return nil, err 171 | } 172 | 173 | func (c *Client) uploadChunck(url string, body io.Reader, size int64, offset int64) (int64, error) { 174 | var method string 175 | 176 | if !c.Config.OverridePatchMethod { 177 | method = "PATCH" 178 | } else { 179 | method = "POST" 180 | } 181 | 182 | req, err := http.NewRequest(method, url, body) 183 | 184 | if err != nil { 185 | return -1, err 186 | } 187 | 188 | req.Header.Set("Content-Type", "application/offset+octet-stream") 189 | req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) 190 | req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10)) 191 | 192 | if c.Config.OverridePatchMethod { 193 | req.Header.Set("X-HTTP-Method-Override", "PATCH") 194 | } 195 | 196 | res, err := c.Do(req) 197 | 198 | if err != nil { 199 | return -1, err 200 | } 201 | defer res.Body.Close() 202 | 203 | switch res.StatusCode { 204 | case 204: 205 | if newOffset, err := strconv.ParseInt(res.Header.Get("Upload-Offset"), 10, 64); err == nil { 206 | return newOffset, nil 207 | } else { 208 | return -1, err 209 | } 210 | case 409: 211 | return -1, ErrOffsetMismatch 212 | case 412: 213 | return -1, ErrVersionMismatch 214 | case 413: 215 | return -1, ErrLargeUpload 216 | default: 217 | return -1, newClientError(res) 218 | } 219 | } 220 | 221 | func (c *Client) getUploadOffset(url string) (int64, error) { 222 | req, err := http.NewRequest("HEAD", url, nil) 223 | 224 | if err != nil { 225 | return -1, err 226 | } 227 | 228 | res, err := c.Do(req) 229 | 230 | if err != nil { 231 | return -1, err 232 | } 233 | defer res.Body.Close() 234 | 235 | switch res.StatusCode { 236 | case 200: 237 | i, err := strconv.ParseInt(res.Header.Get("Upload-Offset"), 10, 64) 238 | 239 | if err == nil { 240 | return i, nil 241 | } else { 242 | return -1, err 243 | } 244 | case 403, 404, 410: 245 | // file doesn't exists. 246 | return -1, ErrUploadNotFound 247 | case 412: 248 | return -1, ErrVersionMismatch 249 | default: 250 | return -1, newClientError(res) 251 | } 252 | } 253 | 254 | func newClientError(res *http.Response) ClientError { 255 | body, _ := ioutil.ReadAll(res.Body) 256 | return ClientError{ 257 | Code: res.StatusCode, 258 | Body: body, 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package tus 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | // Config provides a way to configure the Client depending on your needs. 8 | type Config struct { 9 | // ChunkSize divide the file into chunks. 10 | ChunkSize int64 11 | // Resume enables resumable upload. 12 | Resume bool 13 | // OverridePatchMethod allow to by pass proxies sendind a POST request instead of PATCH. 14 | OverridePatchMethod bool 15 | // Store map an upload's fingerprint with the corresponding upload URL. 16 | // If Resume is true the Store is required. 17 | Store Store 18 | // Set custom header values used in all requests. 19 | Header http.Header 20 | // HTTP Client 21 | HttpClient *http.Client 22 | } 23 | 24 | // DefaultConfig return the default Client configuration. 25 | func DefaultConfig() *Config { 26 | return &Config{ 27 | ChunkSize: 2 * 1024 * 1024, 28 | Resume: false, 29 | OverridePatchMethod: false, 30 | Store: nil, 31 | Header: make(http.Header), 32 | HttpClient: nil, 33 | } 34 | } 35 | 36 | // Validate validates the custom configuration. 37 | func (c *Config) Validate() error { 38 | if c.ChunkSize < 1 { 39 | return ErrChuckSize 40 | } 41 | 42 | if c.Resume && c.Store == nil { 43 | return ErrNilStore 44 | } 45 | 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /config_test.go: -------------------------------------------------------------------------------- 1 | package tus 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestConfingMissingStore(t *testing.T) { 10 | c := Config{ 11 | ChunkSize: 1048576 * 15, // 15 MB 12 | Resume: true, 13 | OverridePatchMethod: false, 14 | Store: nil, 15 | Header: nil, 16 | } 17 | 18 | assert.NotNil(t, c.Validate()) 19 | } 20 | 21 | func TestConfingChunkSizeZero(t *testing.T) { 22 | c := Config{ 23 | ChunkSize: 0, 24 | Resume: false, 25 | OverridePatchMethod: false, 26 | Store: nil, 27 | Header: nil, 28 | } 29 | 30 | assert.NotNil(t, c.Validate()) 31 | } 32 | 33 | func TestConfingValid(t *testing.T) { 34 | c := DefaultConfig() 35 | assert.Nil(t, c.Validate()) 36 | } 37 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package tus provides a client to tus protocol version 1.0.0. 2 | // 3 | // tus is a protocol based on HTTP for resumable file uploads. Resumable means that 4 | // an upload can be interrupted at any moment and can be resumed without 5 | // re-uploading the previous data again. An interruption may happen willingly, if 6 | // the user wants to pause, or by accident in case of an network issue or server 7 | // outage (http://tus.io). 8 | package tus 9 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | app: 4 | build: . 5 | working_dir: /go/src/github.com/eventials/go-tus 6 | command: go run main.go 7 | volumes: 8 | - .:/go/src/github.com/eventials/go-tus 9 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package tus 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | var ( 9 | ErrChuckSize = errors.New("chunk size must be greater than zero.") 10 | ErrNilLogger = errors.New("logger can't be nil.") 11 | ErrNilStore = errors.New("store can't be nil if Resume is enable.") 12 | ErrNilUpload = errors.New("upload can't be nil.") 13 | ErrLargeUpload = errors.New("upload body is to large.") 14 | ErrVersionMismatch = errors.New("protocol version mismatch.") 15 | ErrOffsetMismatch = errors.New("upload offset mismatch.") 16 | ErrUploadNotFound = errors.New("upload not found.") 17 | ErrResumeNotEnabled = errors.New("resuming not enabled.") 18 | ErrFingerprintNotSet = errors.New("fingerprint not set.") 19 | ) 20 | 21 | type ClientError struct { 22 | Code int 23 | Body []byte 24 | } 25 | 26 | func (c ClientError) Error() string { 27 | return fmt.Sprintf("unexpected status code: %d", c.Code) 28 | } 29 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/eventials/go-tus 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/stretchr/testify v1.5.1 7 | github.com/syndtr/goleveldb v1.0.0 8 | github.com/tus/tusd v1.1.0 9 | ) 10 | -------------------------------------------------------------------------------- /leveldbstore/store.go: -------------------------------------------------------------------------------- 1 | package leveldbstore 2 | 3 | import ( 4 | "github.com/eventials/go-tus" 5 | "github.com/syndtr/goleveldb/leveldb" 6 | ) 7 | 8 | type LeveldbStore struct { 9 | db *leveldb.DB 10 | } 11 | 12 | func NewLeveldbStore(path string) (tus.Store, error) { 13 | db, err := leveldb.OpenFile(path, nil) 14 | if err != nil { 15 | return nil, err 16 | } 17 | 18 | store := &LeveldbStore{db: db} 19 | return store, err 20 | } 21 | 22 | func (s *LeveldbStore) Get(fingerprint string) (string, bool) { 23 | url, err := s.db.Get([]byte(fingerprint), nil) 24 | ok := true 25 | if err != nil { 26 | ok = false 27 | } 28 | return string(url), ok 29 | } 30 | 31 | func (s *LeveldbStore) Set(fingerprint, url string) { 32 | s.db.Put([]byte(fingerprint), []byte(url), nil) 33 | } 34 | 35 | func (s *LeveldbStore) Delete(fingerprint string) { 36 | s.db.Delete([]byte(fingerprint), nil) 37 | } 38 | 39 | func (s *LeveldbStore) Close() { 40 | s.Close() 41 | } 42 | -------------------------------------------------------------------------------- /memorystore/store.go: -------------------------------------------------------------------------------- 1 | package memorystore 2 | 3 | import ( 4 | "github.com/eventials/go-tus" 5 | ) 6 | 7 | // MemoryStore implements an in-memory Store. 8 | type MemoryStore struct { 9 | m map[string]string 10 | } 11 | 12 | // NewMemoryStore creates a new MemoryStore. 13 | func NewMemoryStore() (tus.Store, error) { 14 | return &MemoryStore{ 15 | make(map[string]string), 16 | }, nil 17 | } 18 | 19 | func (s *MemoryStore) Get(fingerprint string) (string, bool) { 20 | url, ok := s.m[fingerprint] 21 | return url, ok 22 | } 23 | 24 | func (s *MemoryStore) Set(fingerprint, url string) { 25 | s.m[fingerprint] = url 26 | } 27 | 28 | func (s *MemoryStore) Delete(fingerprint string) { 29 | delete(s.m, fingerprint) 30 | } 31 | 32 | func (s *MemoryStore) Close() { 33 | for k := range s.m { 34 | delete(s.m, k) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /store.go: -------------------------------------------------------------------------------- 1 | package tus 2 | 3 | type Store interface { 4 | Get(fingerprint string) (string, bool) 5 | Set(fingerprint, url string) 6 | Delete(fingerprint string) 7 | Close() 8 | } 9 | -------------------------------------------------------------------------------- /upload.go: -------------------------------------------------------------------------------- 1 | package tus 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "fmt" 7 | "io" 8 | "os" 9 | "strings" 10 | ) 11 | 12 | type Metadata map[string]string 13 | 14 | type Upload struct { 15 | stream io.ReadSeeker 16 | size int64 17 | offset int64 18 | 19 | Fingerprint string 20 | Metadata Metadata 21 | } 22 | 23 | // Updates the Upload information based on offset. 24 | func (u *Upload) updateProgress(offset int64) { 25 | u.offset = offset 26 | } 27 | 28 | // Returns whether this upload is finished or not. 29 | func (u *Upload) Finished() bool { 30 | return u.offset >= u.size 31 | } 32 | 33 | // Returns the progress in a percentage. 34 | func (u *Upload) Progress() int64 { 35 | return (u.offset * 100) / u.size 36 | } 37 | 38 | // Returns the current upload offset. 39 | func (u *Upload) Offset() int64 { 40 | return u.offset 41 | } 42 | 43 | // Returns the size of the upload body. 44 | func (u *Upload) Size() int64 { 45 | return u.size 46 | } 47 | 48 | // EncodedMetadata encodes the upload metadata. 49 | func (u *Upload) EncodedMetadata() string { 50 | var encoded []string 51 | 52 | for k, v := range u.Metadata { 53 | encoded = append(encoded, fmt.Sprintf("%s %s", k, b64encode(v))) 54 | } 55 | 56 | return strings.Join(encoded, ",") 57 | } 58 | 59 | // NewUploadFromFile creates a new Upload from an os.File. 60 | func NewUploadFromFile(f *os.File) (*Upload, error) { 61 | fi, err := f.Stat() 62 | 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | metadata := map[string]string{ 68 | "filename": fi.Name(), 69 | } 70 | 71 | fingerprint := fmt.Sprintf("%s-%d-%s", fi.Name(), fi.Size(), fi.ModTime()) 72 | 73 | return NewUpload(f, fi.Size(), metadata, fingerprint), nil 74 | } 75 | 76 | // NewUploadFromBytes creates a new upload from a byte array. 77 | func NewUploadFromBytes(b []byte) *Upload { 78 | buffer := bytes.NewReader(b) 79 | return NewUpload(buffer, buffer.Size(), nil, "") 80 | } 81 | 82 | // NewUpload creates a new upload from an io.Reader. 83 | func NewUpload(reader io.Reader, size int64, metadata Metadata, fingerprint string) *Upload { 84 | stream, ok := reader.(io.ReadSeeker) 85 | 86 | if !ok { 87 | buf := new(bytes.Buffer) 88 | buf.ReadFrom(reader) 89 | stream = bytes.NewReader(buf.Bytes()) 90 | } 91 | 92 | if metadata == nil { 93 | metadata = make(Metadata) 94 | } 95 | 96 | return &Upload{ 97 | stream: stream, 98 | size: size, 99 | 100 | Fingerprint: fingerprint, 101 | Metadata: metadata, 102 | } 103 | } 104 | 105 | func b64encode(s string) string { 106 | return base64.StdEncoding.EncodeToString([]byte(s)) 107 | } 108 | -------------------------------------------------------------------------------- /upload_test.go: -------------------------------------------------------------------------------- 1 | package tus 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestEncodedMetadata(t *testing.T) { 13 | u := NewUploadFromBytes([]byte("")) 14 | u.Metadata["filename"] = "foobar.txt" 15 | enc := u.EncodedMetadata() 16 | assert.Equal(t, "filename Zm9vYmFyLnR4dA==", enc) 17 | } 18 | 19 | func TestNewUploadFromFile(t *testing.T) { 20 | file := fmt.Sprintf("%s/%d", os.TempDir(), time.Now().Unix()) 21 | 22 | f, err := os.Create(file) 23 | assert.Nil(t, err) 24 | 25 | err = f.Truncate(1048576) // 1 MB 26 | assert.Nil(t, err) 27 | 28 | u, err := NewUploadFromFile(f) 29 | assert.Nil(t, err) 30 | assert.NotNil(t, u) 31 | assert.EqualValues(t, 1048576, u.Size()) 32 | } 33 | -------------------------------------------------------------------------------- /uploader.go: -------------------------------------------------------------------------------- 1 | package tus 2 | 3 | import ( 4 | "bytes" 5 | ) 6 | 7 | type Uploader struct { 8 | client *Client 9 | url string 10 | upload *Upload 11 | offset int64 12 | aborted bool 13 | uploadSubs []chan Upload 14 | notifyChan chan bool 15 | } 16 | 17 | // Subscribes to progress updates. 18 | func (u *Uploader) NotifyUploadProgress(c chan Upload) { 19 | u.uploadSubs = append(u.uploadSubs, c) 20 | } 21 | 22 | // Abort aborts the upload process. 23 | // It doens't abort the current chunck, only the remaining. 24 | func (u *Uploader) Abort() { 25 | u.aborted = true 26 | } 27 | 28 | // IsAborted returns true if the upload was aborted. 29 | func (u *Uploader) IsAborted() bool { 30 | return u.aborted 31 | } 32 | 33 | // Url returns the upload url. 34 | func (u *Uploader) Url() string { 35 | return u.url 36 | } 37 | 38 | // Offset returns the current offset uploaded. 39 | func (u *Uploader) Offset() int64 { 40 | return u.offset 41 | } 42 | 43 | // Upload uploads the entire body to the server. 44 | func (u *Uploader) Upload() error { 45 | for u.offset < u.upload.size && !u.aborted { 46 | err := u.UploadChunck() 47 | 48 | if err != nil { 49 | return err 50 | } 51 | } 52 | 53 | return nil 54 | } 55 | 56 | // UploadChunck uploads a single chunck. 57 | func (u *Uploader) UploadChunck() error { 58 | data := make([]byte, u.client.Config.ChunkSize) 59 | 60 | _, err := u.upload.stream.Seek(u.offset, 0) 61 | 62 | if err != nil { 63 | return err 64 | } 65 | 66 | size, err := u.upload.stream.Read(data) 67 | 68 | if err != nil { 69 | return err 70 | } 71 | 72 | body := bytes.NewBuffer(data[:size]) 73 | 74 | newOffset, err := u.client.uploadChunck(u.url, body, int64(size), u.offset) 75 | 76 | if err != nil { 77 | return err 78 | } 79 | 80 | u.offset = newOffset 81 | 82 | u.upload.updateProgress(u.offset) 83 | 84 | u.notifyChan <- true 85 | 86 | return nil 87 | } 88 | 89 | // Waits for a signal to broadcast to all subscribers 90 | func (u *Uploader) broadcastProgress() { 91 | for _ = range u.notifyChan { 92 | for _, c := range u.uploadSubs { 93 | c <- *u.upload 94 | } 95 | } 96 | } 97 | 98 | // NewUploader creates a new Uploader. 99 | func NewUploader(client *Client, url string, upload *Upload, offset int64) *Uploader { 100 | notifyChan := make(chan bool) 101 | 102 | uploader := &Uploader{ 103 | client, 104 | url, 105 | upload, 106 | offset, 107 | false, 108 | nil, 109 | notifyChan, 110 | } 111 | 112 | go uploader.broadcastProgress() 113 | 114 | return uploader 115 | } 116 | -------------------------------------------------------------------------------- /vendor/github.com/bmizerany/pat/.gitignore: -------------------------------------------------------------------------------- 1 | *.prof 2 | *.out 3 | example/example 4 | -------------------------------------------------------------------------------- /vendor/github.com/bmizerany/pat/README.md: -------------------------------------------------------------------------------- 1 | # pat (formerly pat.go) - A Sinatra style pattern muxer for Go's net/http library 2 | 3 | [![GoDoc](https://godoc.org/github.com/bmizerany/pat?status.svg)](https://godoc.org/github.com/bmizerany/pat) 4 | 5 | ## INSTALL 6 | 7 | $ go get github.com/bmizerany/pat 8 | 9 | ## USE 10 | 11 | ```go 12 | package main 13 | 14 | import ( 15 | "io" 16 | "net/http" 17 | "github.com/bmizerany/pat" 18 | "log" 19 | ) 20 | 21 | // hello world, the web server 22 | func HelloServer(w http.ResponseWriter, req *http.Request) { 23 | io.WriteString(w, "hello, "+req.URL.Query().Get(":name")+"!\n") 24 | } 25 | 26 | func main() { 27 | m := pat.New() 28 | m.Get("/hello/:name", http.HandlerFunc(HelloServer)) 29 | 30 | // Register this pat with the default serve mux so that other packages 31 | // may also be exported. (i.e. /debug/pprof/*) 32 | http.Handle("/", m) 33 | err := http.ListenAndServe(":12345", nil) 34 | if err != nil { 35 | log.Fatal("ListenAndServe: ", err) 36 | } 37 | } 38 | ``` 39 | 40 | It's that simple. 41 | 42 | For more information, see: 43 | http://godoc.org/github.com/bmizerany/pat 44 | 45 | ## CONTRIBUTORS 46 | 47 | * Alexis Svinartchouk (@zvin) 48 | * Blake Mizerany (@bmizerany) 49 | * Brian Ketelsen (@bketelsen) 50 | * Bryan Matsuo (@bmatsuo) 51 | * Caleb Spare (@cespare) 52 | * Evan Shaw (@edsrzf) 53 | * Gary Burd (@garyburd) 54 | * George Rogers (@georgerogers42) 55 | * Keith Rarick (@kr) 56 | * Matt Williams (@mattyw) 57 | * Mike Stipicevic (@wickedchicken) 58 | * Nick Saika (@nesv) 59 | * Timothy Cyrus (@tcyrus) 60 | * binqin (@binku87) 61 | 62 | ## LICENSE 63 | 64 | Copyright (C) 2012 by Keith Rarick, Blake Mizerany 65 | 66 | Permission is hereby granted, free of charge, to any person obtaining a copy 67 | of this software and associated documentation files (the "Software"), to deal 68 | in the Software without restriction, including without limitation the rights 69 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 70 | copies of the Software, and to permit persons to whom the Software is 71 | furnished to do so, subject to the following conditions: 72 | 73 | The above copyright notice and this permission notice shall be included in 74 | all copies or substantial portions of the Software. 75 | 76 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 77 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 78 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 79 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 80 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 81 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 82 | THE SOFTWARE. 83 | -------------------------------------------------------------------------------- /vendor/github.com/davecgh/go-spew/LICENSE: -------------------------------------------------------------------------------- 1 | ISC License 2 | 3 | Copyright (c) 2012-2016 Dave Collins 4 | 5 | Permission to use, copy, modify, and/or distribute this software for any 6 | purpose with or without fee is hereby granted, provided that the above 7 | copyright notice and this permission notice appear in all copies. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | -------------------------------------------------------------------------------- /vendor/github.com/davecgh/go-spew/spew/bypass.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2016 Dave Collins 2 | // 3 | // Permission to use, copy, modify, and distribute this software for any 4 | // purpose with or without fee is hereby granted, provided that the above 5 | // copyright notice and this permission notice appear in all copies. 6 | // 7 | // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | // ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | 15 | // NOTE: Due to the following build constraints, this file will only be compiled 16 | // when the code is not running on Google App Engine, compiled by GopherJS, and 17 | // "-tags safe" is not added to the go build command line. The "disableunsafe" 18 | // tag is deprecated and thus should not be used. 19 | // Go versions prior to 1.4 are disabled because they use a different layout 20 | // for interfaces which make the implementation of unsafeReflectValue more complex. 21 | // +build !js,!appengine,!safe,!disableunsafe,go1.4 22 | 23 | package spew 24 | 25 | import ( 26 | "reflect" 27 | "unsafe" 28 | ) 29 | 30 | const ( 31 | // UnsafeDisabled is a build-time constant which specifies whether or 32 | // not access to the unsafe package is available. 33 | UnsafeDisabled = false 34 | 35 | // ptrSize is the size of a pointer on the current arch. 36 | ptrSize = unsafe.Sizeof((*byte)(nil)) 37 | ) 38 | 39 | type flag uintptr 40 | 41 | var ( 42 | // flagRO indicates whether the value field of a reflect.Value 43 | // is read-only. 44 | flagRO flag 45 | 46 | // flagAddr indicates whether the address of the reflect.Value's 47 | // value may be taken. 48 | flagAddr flag 49 | ) 50 | 51 | // flagKindMask holds the bits that make up the kind 52 | // part of the flags field. In all the supported versions, 53 | // it is in the lower 5 bits. 54 | const flagKindMask = flag(0x1f) 55 | 56 | // Different versions of Go have used different 57 | // bit layouts for the flags type. This table 58 | // records the known combinations. 59 | var okFlags = []struct { 60 | ro, addr flag 61 | }{{ 62 | // From Go 1.4 to 1.5 63 | ro: 1 << 5, 64 | addr: 1 << 7, 65 | }, { 66 | // Up to Go tip. 67 | ro: 1<<5 | 1<<6, 68 | addr: 1 << 8, 69 | }} 70 | 71 | var flagValOffset = func() uintptr { 72 | field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag") 73 | if !ok { 74 | panic("reflect.Value has no flag field") 75 | } 76 | return field.Offset 77 | }() 78 | 79 | // flagField returns a pointer to the flag field of a reflect.Value. 80 | func flagField(v *reflect.Value) *flag { 81 | return (*flag)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + flagValOffset)) 82 | } 83 | 84 | // unsafeReflectValue converts the passed reflect.Value into a one that bypasses 85 | // the typical safety restrictions preventing access to unaddressable and 86 | // unexported data. It works by digging the raw pointer to the underlying 87 | // value out of the protected value and generating a new unprotected (unsafe) 88 | // reflect.Value to it. 89 | // 90 | // This allows us to check for implementations of the Stringer and error 91 | // interfaces to be used for pretty printing ordinarily unaddressable and 92 | // inaccessible values such as unexported struct fields. 93 | func unsafeReflectValue(v reflect.Value) reflect.Value { 94 | if !v.IsValid() || (v.CanInterface() && v.CanAddr()) { 95 | return v 96 | } 97 | flagFieldPtr := flagField(&v) 98 | *flagFieldPtr &^= flagRO 99 | *flagFieldPtr |= flagAddr 100 | return v 101 | } 102 | 103 | // Sanity checks against future reflect package changes 104 | // to the type or semantics of the Value.flag field. 105 | func init() { 106 | field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag") 107 | if !ok { 108 | panic("reflect.Value has no flag field") 109 | } 110 | if field.Type.Kind() != reflect.TypeOf(flag(0)).Kind() { 111 | panic("reflect.Value flag field has changed kind") 112 | } 113 | type t0 int 114 | var t struct { 115 | A t0 116 | // t0 will have flagEmbedRO set. 117 | t0 118 | // a will have flagStickyRO set 119 | a t0 120 | } 121 | vA := reflect.ValueOf(t).FieldByName("A") 122 | va := reflect.ValueOf(t).FieldByName("a") 123 | vt0 := reflect.ValueOf(t).FieldByName("t0") 124 | 125 | // Infer flagRO from the difference between the flags 126 | // for the (otherwise identical) fields in t. 127 | flagPublic := *flagField(&vA) 128 | flagWithRO := *flagField(&va) | *flagField(&vt0) 129 | flagRO = flagPublic ^ flagWithRO 130 | 131 | // Infer flagAddr from the difference between a value 132 | // taken from a pointer and not. 133 | vPtrA := reflect.ValueOf(&t).Elem().FieldByName("A") 134 | flagNoPtr := *flagField(&vA) 135 | flagPtr := *flagField(&vPtrA) 136 | flagAddr = flagNoPtr ^ flagPtr 137 | 138 | // Check that the inferred flags tally with one of the known versions. 139 | for _, f := range okFlags { 140 | if flagRO == f.ro && flagAddr == f.addr { 141 | return 142 | } 143 | } 144 | panic("reflect.Value read-only flag has changed semantics") 145 | } 146 | -------------------------------------------------------------------------------- /vendor/github.com/davecgh/go-spew/spew/bypasssafe.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015-2016 Dave Collins 2 | // 3 | // Permission to use, copy, modify, and distribute this software for any 4 | // purpose with or without fee is hereby granted, provided that the above 5 | // copyright notice and this permission notice appear in all copies. 6 | // 7 | // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | // ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | // ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | 15 | // NOTE: Due to the following build constraints, this file will only be compiled 16 | // when the code is running on Google App Engine, compiled by GopherJS, or 17 | // "-tags safe" is added to the go build command line. The "disableunsafe" 18 | // tag is deprecated and thus should not be used. 19 | // +build js appengine safe disableunsafe !go1.4 20 | 21 | package spew 22 | 23 | import "reflect" 24 | 25 | const ( 26 | // UnsafeDisabled is a build-time constant which specifies whether or 27 | // not access to the unsafe package is available. 28 | UnsafeDisabled = true 29 | ) 30 | 31 | // unsafeReflectValue typically converts the passed reflect.Value into a one 32 | // that bypasses the typical safety restrictions preventing access to 33 | // unaddressable and unexported data. However, doing this relies on access to 34 | // the unsafe package. This is a stub version which simply returns the passed 35 | // reflect.Value when the unsafe package is not available. 36 | func unsafeReflectValue(v reflect.Value) reflect.Value { 37 | return v 38 | } 39 | -------------------------------------------------------------------------------- /vendor/github.com/davecgh/go-spew/spew/spew.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2013-2016 Dave Collins 3 | * 4 | * Permission to use, copy, modify, and distribute this software for any 5 | * purpose with or without fee is hereby granted, provided that the above 6 | * copyright notice and this permission notice appear in all copies. 7 | * 8 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 9 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 10 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 11 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 12 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 13 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 14 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 15 | */ 16 | 17 | package spew 18 | 19 | import ( 20 | "fmt" 21 | "io" 22 | ) 23 | 24 | // Errorf is a wrapper for fmt.Errorf that treats each argument as if it were 25 | // passed with a default Formatter interface returned by NewFormatter. It 26 | // returns the formatted string as a value that satisfies error. See 27 | // NewFormatter for formatting details. 28 | // 29 | // This function is shorthand for the following syntax: 30 | // 31 | // fmt.Errorf(format, spew.NewFormatter(a), spew.NewFormatter(b)) 32 | func Errorf(format string, a ...interface{}) (err error) { 33 | return fmt.Errorf(format, convertArgs(a)...) 34 | } 35 | 36 | // Fprint is a wrapper for fmt.Fprint that treats each argument as if it were 37 | // passed with a default Formatter interface returned by NewFormatter. It 38 | // returns the number of bytes written and any write error encountered. See 39 | // NewFormatter for formatting details. 40 | // 41 | // This function is shorthand for the following syntax: 42 | // 43 | // fmt.Fprint(w, spew.NewFormatter(a), spew.NewFormatter(b)) 44 | func Fprint(w io.Writer, a ...interface{}) (n int, err error) { 45 | return fmt.Fprint(w, convertArgs(a)...) 46 | } 47 | 48 | // Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were 49 | // passed with a default Formatter interface returned by NewFormatter. It 50 | // returns the number of bytes written and any write error encountered. See 51 | // NewFormatter for formatting details. 52 | // 53 | // This function is shorthand for the following syntax: 54 | // 55 | // fmt.Fprintf(w, format, spew.NewFormatter(a), spew.NewFormatter(b)) 56 | func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) { 57 | return fmt.Fprintf(w, format, convertArgs(a)...) 58 | } 59 | 60 | // Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it 61 | // passed with a default Formatter interface returned by NewFormatter. See 62 | // NewFormatter for formatting details. 63 | // 64 | // This function is shorthand for the following syntax: 65 | // 66 | // fmt.Fprintln(w, spew.NewFormatter(a), spew.NewFormatter(b)) 67 | func Fprintln(w io.Writer, a ...interface{}) (n int, err error) { 68 | return fmt.Fprintln(w, convertArgs(a)...) 69 | } 70 | 71 | // Print is a wrapper for fmt.Print that treats each argument as if it were 72 | // passed with a default Formatter interface returned by NewFormatter. It 73 | // returns the number of bytes written and any write error encountered. See 74 | // NewFormatter for formatting details. 75 | // 76 | // This function is shorthand for the following syntax: 77 | // 78 | // fmt.Print(spew.NewFormatter(a), spew.NewFormatter(b)) 79 | func Print(a ...interface{}) (n int, err error) { 80 | return fmt.Print(convertArgs(a)...) 81 | } 82 | 83 | // Printf is a wrapper for fmt.Printf that treats each argument as if it were 84 | // passed with a default Formatter interface returned by NewFormatter. It 85 | // returns the number of bytes written and any write error encountered. See 86 | // NewFormatter for formatting details. 87 | // 88 | // This function is shorthand for the following syntax: 89 | // 90 | // fmt.Printf(format, spew.NewFormatter(a), spew.NewFormatter(b)) 91 | func Printf(format string, a ...interface{}) (n int, err error) { 92 | return fmt.Printf(format, convertArgs(a)...) 93 | } 94 | 95 | // Println is a wrapper for fmt.Println that treats each argument as if it were 96 | // passed with a default Formatter interface returned by NewFormatter. It 97 | // returns the number of bytes written and any write error encountered. See 98 | // NewFormatter for formatting details. 99 | // 100 | // This function is shorthand for the following syntax: 101 | // 102 | // fmt.Println(spew.NewFormatter(a), spew.NewFormatter(b)) 103 | func Println(a ...interface{}) (n int, err error) { 104 | return fmt.Println(convertArgs(a)...) 105 | } 106 | 107 | // Sprint is a wrapper for fmt.Sprint that treats each argument as if it were 108 | // passed with a default Formatter interface returned by NewFormatter. It 109 | // returns the resulting string. See NewFormatter for formatting details. 110 | // 111 | // This function is shorthand for the following syntax: 112 | // 113 | // fmt.Sprint(spew.NewFormatter(a), spew.NewFormatter(b)) 114 | func Sprint(a ...interface{}) string { 115 | return fmt.Sprint(convertArgs(a)...) 116 | } 117 | 118 | // Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were 119 | // passed with a default Formatter interface returned by NewFormatter. It 120 | // returns the resulting string. See NewFormatter for formatting details. 121 | // 122 | // This function is shorthand for the following syntax: 123 | // 124 | // fmt.Sprintf(format, spew.NewFormatter(a), spew.NewFormatter(b)) 125 | func Sprintf(format string, a ...interface{}) string { 126 | return fmt.Sprintf(format, convertArgs(a)...) 127 | } 128 | 129 | // Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it 130 | // were passed with a default Formatter interface returned by NewFormatter. It 131 | // returns the resulting string. See NewFormatter for formatting details. 132 | // 133 | // This function is shorthand for the following syntax: 134 | // 135 | // fmt.Sprintln(spew.NewFormatter(a), spew.NewFormatter(b)) 136 | func Sprintln(a ...interface{}) string { 137 | return fmt.Sprintln(convertArgs(a)...) 138 | } 139 | 140 | // convertArgs accepts a slice of arguments and returns a slice of the same 141 | // length with each argument converted to a default spew Formatter interface. 142 | func convertArgs(args []interface{}) (formatters []interface{}) { 143 | formatters = make([]interface{}, len(args)) 144 | for index, arg := range args { 145 | formatters[index] = NewFormatter(arg) 146 | } 147 | return formatters 148 | } 149 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/.gitignore: -------------------------------------------------------------------------------- 1 | cmd/snappytool/snappytool 2 | testdata/bench 3 | 4 | # These explicitly listed benchmark data files are for an obsolete version of 5 | # snappy_test.go. 6 | testdata/alice29.txt 7 | testdata/asyoulik.txt 8 | testdata/fireworks.jpeg 9 | testdata/geo.protodata 10 | testdata/html 11 | testdata/html_x_4 12 | testdata/kppkn.gtb 13 | testdata/lcet10.txt 14 | testdata/paper-100k.pdf 15 | testdata/plrabn12.txt 16 | testdata/urls.10K 17 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of Snappy-Go authors for copyright purposes. 2 | # This file is distinct from the CONTRIBUTORS files. 3 | # See the latter for an explanation. 4 | 5 | # Names should be added to this file as 6 | # Name or Organization 7 | # The email address is not required for organizations. 8 | 9 | # Please keep the list sorted. 10 | 11 | Damian Gryski 12 | Google Inc. 13 | Jan Mercl <0xjnml@gmail.com> 14 | Rodolfo Carvalho 15 | Sebastien Binet 16 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # This is the official list of people who can contribute 2 | # (and typically have contributed) code to the Snappy-Go repository. 3 | # The AUTHORS file lists the copyright holders; this file 4 | # lists people. For example, Google employees are listed here 5 | # but not in AUTHORS, because Google holds the copyright. 6 | # 7 | # The submission process automatically checks to make sure 8 | # that people submitting code are listed in this file (by email address). 9 | # 10 | # Names should be added to this file only after verifying that 11 | # the individual or the individual's organization has agreed to 12 | # the appropriate Contributor License Agreement, found here: 13 | # 14 | # http://code.google.com/legal/individual-cla-v1.0.html 15 | # http://code.google.com/legal/corporate-cla-v1.0.html 16 | # 17 | # The agreement for individuals can be filled out on the web. 18 | # 19 | # When adding J Random Contributor's name to this file, 20 | # either J's name or J's organization's name should be 21 | # added to the AUTHORS file, depending on whether the 22 | # individual or corporate CLA was used. 23 | 24 | # Names should be added to this file like so: 25 | # Name 26 | 27 | # Please keep the list sorted. 28 | 29 | Damian Gryski 30 | Jan Mercl <0xjnml@gmail.com> 31 | Kai Backman 32 | Marc-Antoine Ruel 33 | Nigel Tao 34 | Rob Pike 35 | Rodolfo Carvalho 36 | Russ Cox 37 | Sebastien Binet 38 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/README: -------------------------------------------------------------------------------- 1 | The Snappy compression format in the Go programming language. 2 | 3 | To download and install from source: 4 | $ go get github.com/golang/snappy 5 | 6 | Unless otherwise noted, the Snappy-Go source files are distributed 7 | under the BSD-style license found in the LICENSE file. 8 | 9 | 10 | 11 | Benchmarks. 12 | 13 | The golang/snappy benchmarks include compressing (Z) and decompressing (U) ten 14 | or so files, the same set used by the C++ Snappy code (github.com/google/snappy 15 | and note the "google", not "golang"). On an "Intel(R) Core(TM) i7-3770 CPU @ 16 | 3.40GHz", Go's GOARCH=amd64 numbers as of 2016-05-29: 17 | 18 | "go test -test.bench=." 19 | 20 | _UFlat0-8 2.19GB/s ± 0% html 21 | _UFlat1-8 1.41GB/s ± 0% urls 22 | _UFlat2-8 23.5GB/s ± 2% jpg 23 | _UFlat3-8 1.91GB/s ± 0% jpg_200 24 | _UFlat4-8 14.0GB/s ± 1% pdf 25 | _UFlat5-8 1.97GB/s ± 0% html4 26 | _UFlat6-8 814MB/s ± 0% txt1 27 | _UFlat7-8 785MB/s ± 0% txt2 28 | _UFlat8-8 857MB/s ± 0% txt3 29 | _UFlat9-8 719MB/s ± 1% txt4 30 | _UFlat10-8 2.84GB/s ± 0% pb 31 | _UFlat11-8 1.05GB/s ± 0% gaviota 32 | 33 | _ZFlat0-8 1.04GB/s ± 0% html 34 | _ZFlat1-8 534MB/s ± 0% urls 35 | _ZFlat2-8 15.7GB/s ± 1% jpg 36 | _ZFlat3-8 740MB/s ± 3% jpg_200 37 | _ZFlat4-8 9.20GB/s ± 1% pdf 38 | _ZFlat5-8 991MB/s ± 0% html4 39 | _ZFlat6-8 379MB/s ± 0% txt1 40 | _ZFlat7-8 352MB/s ± 0% txt2 41 | _ZFlat8-8 396MB/s ± 1% txt3 42 | _ZFlat9-8 327MB/s ± 1% txt4 43 | _ZFlat10-8 1.33GB/s ± 1% pb 44 | _ZFlat11-8 605MB/s ± 1% gaviota 45 | 46 | 47 | 48 | "go test -test.bench=. -tags=noasm" 49 | 50 | _UFlat0-8 621MB/s ± 2% html 51 | _UFlat1-8 494MB/s ± 1% urls 52 | _UFlat2-8 23.2GB/s ± 1% jpg 53 | _UFlat3-8 1.12GB/s ± 1% jpg_200 54 | _UFlat4-8 4.35GB/s ± 1% pdf 55 | _UFlat5-8 609MB/s ± 0% html4 56 | _UFlat6-8 296MB/s ± 0% txt1 57 | _UFlat7-8 288MB/s ± 0% txt2 58 | _UFlat8-8 309MB/s ± 1% txt3 59 | _UFlat9-8 280MB/s ± 1% txt4 60 | _UFlat10-8 753MB/s ± 0% pb 61 | _UFlat11-8 400MB/s ± 0% gaviota 62 | 63 | _ZFlat0-8 409MB/s ± 1% html 64 | _ZFlat1-8 250MB/s ± 1% urls 65 | _ZFlat2-8 12.3GB/s ± 1% jpg 66 | _ZFlat3-8 132MB/s ± 0% jpg_200 67 | _ZFlat4-8 2.92GB/s ± 0% pdf 68 | _ZFlat5-8 405MB/s ± 1% html4 69 | _ZFlat6-8 179MB/s ± 1% txt1 70 | _ZFlat7-8 170MB/s ± 1% txt2 71 | _ZFlat8-8 189MB/s ± 1% txt3 72 | _ZFlat9-8 164MB/s ± 1% txt4 73 | _ZFlat10-8 479MB/s ± 1% pb 74 | _ZFlat11-8 270MB/s ± 1% gaviota 75 | 76 | 77 | 78 | For comparison (Go's encoded output is byte-for-byte identical to C++'s), here 79 | are the numbers from C++ Snappy's 80 | 81 | make CXXFLAGS="-O2 -DNDEBUG -g" clean snappy_unittest.log && cat snappy_unittest.log 82 | 83 | BM_UFlat/0 2.4GB/s html 84 | BM_UFlat/1 1.4GB/s urls 85 | BM_UFlat/2 21.8GB/s jpg 86 | BM_UFlat/3 1.5GB/s jpg_200 87 | BM_UFlat/4 13.3GB/s pdf 88 | BM_UFlat/5 2.1GB/s html4 89 | BM_UFlat/6 1.0GB/s txt1 90 | BM_UFlat/7 959.4MB/s txt2 91 | BM_UFlat/8 1.0GB/s txt3 92 | BM_UFlat/9 864.5MB/s txt4 93 | BM_UFlat/10 2.9GB/s pb 94 | BM_UFlat/11 1.2GB/s gaviota 95 | 96 | BM_ZFlat/0 944.3MB/s html (22.31 %) 97 | BM_ZFlat/1 501.6MB/s urls (47.78 %) 98 | BM_ZFlat/2 14.3GB/s jpg (99.95 %) 99 | BM_ZFlat/3 538.3MB/s jpg_200 (73.00 %) 100 | BM_ZFlat/4 8.3GB/s pdf (83.30 %) 101 | BM_ZFlat/5 903.5MB/s html4 (22.52 %) 102 | BM_ZFlat/6 336.0MB/s txt1 (57.88 %) 103 | BM_ZFlat/7 312.3MB/s txt2 (61.91 %) 104 | BM_ZFlat/8 353.1MB/s txt3 (54.99 %) 105 | BM_ZFlat/9 289.9MB/s txt4 (66.26 %) 106 | BM_ZFlat/10 1.2GB/s pb (19.68 %) 107 | BM_ZFlat/11 527.4MB/s gaviota (37.72 %) 108 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/decode_amd64.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Snappy-Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !appengine 6 | // +build gc 7 | // +build !noasm 8 | 9 | package snappy 10 | 11 | // decode has the same semantics as in decode_other.go. 12 | // 13 | //go:noescape 14 | func decode(dst, src []byte) int 15 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/decode_other.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Snappy-Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !amd64 appengine !gc noasm 6 | 7 | package snappy 8 | 9 | // decode writes the decoding of src to dst. It assumes that the varint-encoded 10 | // length of the decompressed bytes has already been read, and that len(dst) 11 | // equals that length. 12 | // 13 | // It returns 0 on success or a decodeErrCodeXxx error code on failure. 14 | func decode(dst, src []byte) int { 15 | var d, s, offset, length int 16 | for s < len(src) { 17 | switch src[s] & 0x03 { 18 | case tagLiteral: 19 | x := uint32(src[s] >> 2) 20 | switch { 21 | case x < 60: 22 | s++ 23 | case x == 60: 24 | s += 2 25 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 26 | return decodeErrCodeCorrupt 27 | } 28 | x = uint32(src[s-1]) 29 | case x == 61: 30 | s += 3 31 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 32 | return decodeErrCodeCorrupt 33 | } 34 | x = uint32(src[s-2]) | uint32(src[s-1])<<8 35 | case x == 62: 36 | s += 4 37 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 38 | return decodeErrCodeCorrupt 39 | } 40 | x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 41 | case x == 63: 42 | s += 5 43 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 44 | return decodeErrCodeCorrupt 45 | } 46 | x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 47 | } 48 | length = int(x) + 1 49 | if length <= 0 { 50 | return decodeErrCodeUnsupportedLiteralLength 51 | } 52 | if length > len(dst)-d || length > len(src)-s { 53 | return decodeErrCodeCorrupt 54 | } 55 | copy(dst[d:], src[s:s+length]) 56 | d += length 57 | s += length 58 | continue 59 | 60 | case tagCopy1: 61 | s += 2 62 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 63 | return decodeErrCodeCorrupt 64 | } 65 | length = 4 + int(src[s-2])>>2&0x7 66 | offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) 67 | 68 | case tagCopy2: 69 | s += 3 70 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 71 | return decodeErrCodeCorrupt 72 | } 73 | length = 1 + int(src[s-3])>>2 74 | offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) 75 | 76 | case tagCopy4: 77 | s += 5 78 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 79 | return decodeErrCodeCorrupt 80 | } 81 | length = 1 + int(src[s-5])>>2 82 | offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) 83 | } 84 | 85 | if offset <= 0 || d < offset || length > len(dst)-d { 86 | return decodeErrCodeCorrupt 87 | } 88 | // Copy from an earlier sub-slice of dst to a later sub-slice. Unlike 89 | // the built-in copy function, this byte-by-byte copy always runs 90 | // forwards, even if the slices overlap. Conceptually, this is: 91 | // 92 | // d += forwardCopy(dst[d:d+length], dst[d-offset:]) 93 | for end := d + length; d != end; d++ { 94 | dst[d] = dst[d-offset] 95 | } 96 | } 97 | if d != len(dst) { 98 | return decodeErrCodeCorrupt 99 | } 100 | return 0 101 | } 102 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/encode_amd64.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Snappy-Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !appengine 6 | // +build gc 7 | // +build !noasm 8 | 9 | package snappy 10 | 11 | // emitLiteral has the same semantics as in encode_other.go. 12 | // 13 | //go:noescape 14 | func emitLiteral(dst, lit []byte) int 15 | 16 | // emitCopy has the same semantics as in encode_other.go. 17 | // 18 | //go:noescape 19 | func emitCopy(dst []byte, offset, length int) int 20 | 21 | // extendMatch has the same semantics as in encode_other.go. 22 | // 23 | //go:noescape 24 | func extendMatch(src []byte, i, j int) int 25 | 26 | // encodeBlock has the same semantics as in encode_other.go. 27 | // 28 | //go:noescape 29 | func encodeBlock(dst, src []byte) (d int) 30 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/snappy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Snappy-Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package snappy implements the Snappy compression format. It aims for very 6 | // high speeds and reasonable compression. 7 | // 8 | // There are actually two Snappy formats: block and stream. They are related, 9 | // but different: trying to decompress block-compressed data as a Snappy stream 10 | // will fail, and vice versa. The block format is the Decode and Encode 11 | // functions and the stream format is the Reader and Writer types. 12 | // 13 | // The block format, the more common case, is used when the complete size (the 14 | // number of bytes) of the original data is known upfront, at the time 15 | // compression starts. The stream format, also known as the framing format, is 16 | // for when that isn't always true. 17 | // 18 | // The canonical, C++ implementation is at https://github.com/google/snappy and 19 | // it only implements the block format. 20 | package snappy // import "github.com/golang/snappy" 21 | 22 | import ( 23 | "hash/crc32" 24 | ) 25 | 26 | /* 27 | Each encoded block begins with the varint-encoded length of the decoded data, 28 | followed by a sequence of chunks. Chunks begin and end on byte boundaries. The 29 | first byte of each chunk is broken into its 2 least and 6 most significant bits 30 | called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag. 31 | Zero means a literal tag. All other values mean a copy tag. 32 | 33 | For literal tags: 34 | - If m < 60, the next 1 + m bytes are literal bytes. 35 | - Otherwise, let n be the little-endian unsigned integer denoted by the next 36 | m - 59 bytes. The next 1 + n bytes after that are literal bytes. 37 | 38 | For copy tags, length bytes are copied from offset bytes ago, in the style of 39 | Lempel-Ziv compression algorithms. In particular: 40 | - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12). 41 | The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10 42 | of the offset. The next byte is bits 0-7 of the offset. 43 | - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65). 44 | The length is 1 + m. The offset is the little-endian unsigned integer 45 | denoted by the next 2 bytes. 46 | - For l == 3, this tag is a legacy format that is no longer issued by most 47 | encoders. Nonetheless, the offset ranges in [0, 1<<32) and the length in 48 | [1, 65). The length is 1 + m. The offset is the little-endian unsigned 49 | integer denoted by the next 4 bytes. 50 | */ 51 | const ( 52 | tagLiteral = 0x00 53 | tagCopy1 = 0x01 54 | tagCopy2 = 0x02 55 | tagCopy4 = 0x03 56 | ) 57 | 58 | const ( 59 | checksumSize = 4 60 | chunkHeaderSize = 4 61 | magicChunk = "\xff\x06\x00\x00" + magicBody 62 | magicBody = "sNaPpY" 63 | 64 | // maxBlockSize is the maximum size of the input to encodeBlock. It is not 65 | // part of the wire format per se, but some parts of the encoder assume 66 | // that an offset fits into a uint16. 67 | // 68 | // Also, for the framing format (Writer type instead of Encode function), 69 | // https://github.com/google/snappy/blob/master/framing_format.txt says 70 | // that "the uncompressed data in a chunk must be no longer than 65536 71 | // bytes". 72 | maxBlockSize = 65536 73 | 74 | // maxEncodedLenOfMaxBlockSize equals MaxEncodedLen(maxBlockSize), but is 75 | // hard coded to be a const instead of a variable, so that obufLen can also 76 | // be a const. Their equivalence is confirmed by 77 | // TestMaxEncodedLenOfMaxBlockSize. 78 | maxEncodedLenOfMaxBlockSize = 76490 79 | 80 | obufHeaderLen = len(magicChunk) + checksumSize + chunkHeaderSize 81 | obufLen = obufHeaderLen + maxEncodedLenOfMaxBlockSize 82 | ) 83 | 84 | const ( 85 | chunkTypeCompressedData = 0x00 86 | chunkTypeUncompressedData = 0x01 87 | chunkTypePadding = 0xfe 88 | chunkTypeStreamIdentifier = 0xff 89 | ) 90 | 91 | var crcTable = crc32.MakeTable(crc32.Castagnoli) 92 | 93 | // crc implements the checksum specified in section 3 of 94 | // https://github.com/google/snappy/blob/master/framing_format.txt 95 | func crc(b []byte) uint32 { 96 | c := crc32.Update(0, crcTable, b) 97 | return uint32(c>>15|c<<17) + 0xa282ead8 98 | } 99 | -------------------------------------------------------------------------------- /vendor/github.com/pmezard/go-difflib/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, Patrick Mezard 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | The names of its contributors may not be used to endorse or promote 14 | products derived from this software without specific prior written 15 | permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 18 | IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 | TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 20 | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 23 | TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 24 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 25 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 26 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2012-2018 Mat Ryer and Tyler Bunnell 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/assert/assertion_format.go.tmpl: -------------------------------------------------------------------------------- 1 | {{.CommentFormat}} 2 | func {{.DocInfo.Name}}f(t TestingT, {{.ParamsFormat}}) bool { 3 | if h, ok := t.(tHelper); ok { h.Helper() } 4 | return {{.DocInfo.Name}}(t, {{.ForwardedParamsFormat}}) 5 | } 6 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/assert/assertion_forward.go.tmpl: -------------------------------------------------------------------------------- 1 | {{.CommentWithoutT "a"}} 2 | func (a *Assertions) {{.DocInfo.Name}}({{.Params}}) bool { 3 | if h, ok := a.t.(tHelper); ok { h.Helper() } 4 | return {{.DocInfo.Name}}(a.t, {{.ForwardedParams}}) 5 | } 6 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/assert/doc.go: -------------------------------------------------------------------------------- 1 | // Package assert provides a set of comprehensive testing tools for use with the normal Go testing system. 2 | // 3 | // Example Usage 4 | // 5 | // The following is a complete example using assert in a standard test function: 6 | // import ( 7 | // "testing" 8 | // "github.com/stretchr/testify/assert" 9 | // ) 10 | // 11 | // func TestSomething(t *testing.T) { 12 | // 13 | // var a string = "Hello" 14 | // var b string = "Hello" 15 | // 16 | // assert.Equal(t, a, b, "The two words should be the same.") 17 | // 18 | // } 19 | // 20 | // if you assert many times, use the format below: 21 | // 22 | // import ( 23 | // "testing" 24 | // "github.com/stretchr/testify/assert" 25 | // ) 26 | // 27 | // func TestSomething(t *testing.T) { 28 | // assert := assert.New(t) 29 | // 30 | // var a string = "Hello" 31 | // var b string = "Hello" 32 | // 33 | // assert.Equal(a, b, "The two words should be the same.") 34 | // } 35 | // 36 | // Assertions 37 | // 38 | // Assertions allow you to easily write test code, and are global funcs in the `assert` package. 39 | // All assertion functions take, as the first argument, the `*testing.T` object provided by the 40 | // testing framework. This allows the assertion funcs to write the failings and other details to 41 | // the correct place. 42 | // 43 | // Every assertion function also takes an optional string message as the final argument, 44 | // allowing custom error messages to be appended to the message the assertion method outputs. 45 | package assert 46 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/assert/errors.go: -------------------------------------------------------------------------------- 1 | package assert 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | // AnError is an error instance useful for testing. If the code does not care 8 | // about error specifics, and only needs to return the error for example, this 9 | // error should be used to make the test code more readable. 10 | var AnError = errors.New("assert.AnError general error for testing") 11 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/assert/forward_assertions.go: -------------------------------------------------------------------------------- 1 | package assert 2 | 3 | // Assertions provides assertion methods around the 4 | // TestingT interface. 5 | type Assertions struct { 6 | t TestingT 7 | } 8 | 9 | // New makes a new Assertions object for the specified TestingT. 10 | func New(t TestingT) *Assertions { 11 | return &Assertions{ 12 | t: t, 13 | } 14 | } 15 | 16 | //go:generate sh -c "cd ../_codegen && go build && cd - && ../_codegen/_codegen -output-package=assert -template=assertion_forward.go.tmpl -include-format-funcs" 17 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/assert/http_assertions.go: -------------------------------------------------------------------------------- 1 | package assert 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | "net/url" 8 | "strings" 9 | ) 10 | 11 | // httpCode is a helper that returns HTTP code of the response. It returns -1 and 12 | // an error if building a new request fails. 13 | func httpCode(handler http.HandlerFunc, method, url string, values url.Values) (int, error) { 14 | w := httptest.NewRecorder() 15 | req, err := http.NewRequest(method, url, nil) 16 | if err != nil { 17 | return -1, err 18 | } 19 | req.URL.RawQuery = values.Encode() 20 | handler(w, req) 21 | return w.Code, nil 22 | } 23 | 24 | // HTTPSuccess asserts that a specified handler returns a success status code. 25 | // 26 | // assert.HTTPSuccess(t, myHandler, "POST", "http://www.google.com", nil) 27 | // 28 | // Returns whether the assertion was successful (true) or not (false). 29 | func HTTPSuccess(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool { 30 | if h, ok := t.(tHelper); ok { 31 | h.Helper() 32 | } 33 | code, err := httpCode(handler, method, url, values) 34 | if err != nil { 35 | Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err)) 36 | return false 37 | } 38 | 39 | isSuccessCode := code >= http.StatusOK && code <= http.StatusPartialContent 40 | if !isSuccessCode { 41 | Fail(t, fmt.Sprintf("Expected HTTP success status code for %q but received %d", url+"?"+values.Encode(), code)) 42 | } 43 | 44 | return isSuccessCode 45 | } 46 | 47 | // HTTPRedirect asserts that a specified handler returns a redirect status code. 48 | // 49 | // assert.HTTPRedirect(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} 50 | // 51 | // Returns whether the assertion was successful (true) or not (false). 52 | func HTTPRedirect(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool { 53 | if h, ok := t.(tHelper); ok { 54 | h.Helper() 55 | } 56 | code, err := httpCode(handler, method, url, values) 57 | if err != nil { 58 | Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err)) 59 | return false 60 | } 61 | 62 | isRedirectCode := code >= http.StatusMultipleChoices && code <= http.StatusTemporaryRedirect 63 | if !isRedirectCode { 64 | Fail(t, fmt.Sprintf("Expected HTTP redirect status code for %q but received %d", url+"?"+values.Encode(), code)) 65 | } 66 | 67 | return isRedirectCode 68 | } 69 | 70 | // HTTPError asserts that a specified handler returns an error status code. 71 | // 72 | // assert.HTTPError(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} 73 | // 74 | // Returns whether the assertion was successful (true) or not (false). 75 | func HTTPError(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, msgAndArgs ...interface{}) bool { 76 | if h, ok := t.(tHelper); ok { 77 | h.Helper() 78 | } 79 | code, err := httpCode(handler, method, url, values) 80 | if err != nil { 81 | Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err)) 82 | return false 83 | } 84 | 85 | isErrorCode := code >= http.StatusBadRequest 86 | if !isErrorCode { 87 | Fail(t, fmt.Sprintf("Expected HTTP error status code for %q but received %d", url+"?"+values.Encode(), code)) 88 | } 89 | 90 | return isErrorCode 91 | } 92 | 93 | // HTTPBody is a helper that returns HTTP body of the response. It returns 94 | // empty string if building a new request fails. 95 | func HTTPBody(handler http.HandlerFunc, method, url string, values url.Values) string { 96 | w := httptest.NewRecorder() 97 | req, err := http.NewRequest(method, url+"?"+values.Encode(), nil) 98 | if err != nil { 99 | return "" 100 | } 101 | handler(w, req) 102 | return w.Body.String() 103 | } 104 | 105 | // HTTPBodyContains asserts that a specified handler returns a 106 | // body that contains a string. 107 | // 108 | // assert.HTTPBodyContains(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") 109 | // 110 | // Returns whether the assertion was successful (true) or not (false). 111 | func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { 112 | if h, ok := t.(tHelper); ok { 113 | h.Helper() 114 | } 115 | body := HTTPBody(handler, method, url, values) 116 | 117 | contains := strings.Contains(body, fmt.Sprint(str)) 118 | if !contains { 119 | Fail(t, fmt.Sprintf("Expected response body for \"%s\" to contain \"%s\" but found \"%s\"", url+"?"+values.Encode(), str, body)) 120 | } 121 | 122 | return contains 123 | } 124 | 125 | // HTTPBodyNotContains asserts that a specified handler returns a 126 | // body that does not contain a string. 127 | // 128 | // assert.HTTPBodyNotContains(t, myHandler, "GET", "www.google.com", nil, "I'm Feeling Lucky") 129 | // 130 | // Returns whether the assertion was successful (true) or not (false). 131 | func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, str interface{}, msgAndArgs ...interface{}) bool { 132 | if h, ok := t.(tHelper); ok { 133 | h.Helper() 134 | } 135 | body := HTTPBody(handler, method, url, values) 136 | 137 | contains := strings.Contains(body, fmt.Sprint(str)) 138 | if contains { 139 | Fail(t, fmt.Sprintf("Expected response body for \"%s\" to NOT contain \"%s\" but found \"%s\"", url+"?"+values.Encode(), str, body)) 140 | } 141 | 142 | return !contains 143 | } 144 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/require/doc.go: -------------------------------------------------------------------------------- 1 | // Package require implements the same assertions as the `assert` package but 2 | // stops test execution when a test fails. 3 | // 4 | // Example Usage 5 | // 6 | // The following is a complete example using require in a standard test function: 7 | // import ( 8 | // "testing" 9 | // "github.com/stretchr/testify/require" 10 | // ) 11 | // 12 | // func TestSomething(t *testing.T) { 13 | // 14 | // var a string = "Hello" 15 | // var b string = "Hello" 16 | // 17 | // require.Equal(t, a, b, "The two words should be the same.") 18 | // 19 | // } 20 | // 21 | // Assertions 22 | // 23 | // The `require` package have same global functions as in the `assert` package, 24 | // but instead of returning a boolean result they call `t.FailNow()`. 25 | // 26 | // Every assertion function also takes an optional string message as the final argument, 27 | // allowing custom error messages to be appended to the message the assertion method outputs. 28 | package require 29 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/require/forward_requirements.go: -------------------------------------------------------------------------------- 1 | package require 2 | 3 | // Assertions provides assertion methods around the 4 | // TestingT interface. 5 | type Assertions struct { 6 | t TestingT 7 | } 8 | 9 | // New makes a new Assertions object for the specified TestingT. 10 | func New(t TestingT) *Assertions { 11 | return &Assertions{ 12 | t: t, 13 | } 14 | } 15 | 16 | //go:generate sh -c "cd ../_codegen && go build && cd - && ../_codegen/_codegen -output-package=require -template=require_forward.go.tmpl -include-format-funcs" 17 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/require/require.go.tmpl: -------------------------------------------------------------------------------- 1 | {{.Comment}} 2 | func {{.DocInfo.Name}}(t TestingT, {{.Params}}) { 3 | if h, ok := t.(tHelper); ok { h.Helper() } 4 | if assert.{{.DocInfo.Name}}(t, {{.ForwardedParams}}) { return } 5 | t.FailNow() 6 | } 7 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/require/require_forward.go.tmpl: -------------------------------------------------------------------------------- 1 | {{.CommentWithoutT "a"}} 2 | func (a *Assertions) {{.DocInfo.Name}}({{.Params}}) { 3 | if h, ok := a.t.(tHelper); ok { h.Helper() } 4 | {{.DocInfo.Name}}(a.t, {{.ForwardedParams}}) 5 | } 6 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/require/requirements.go: -------------------------------------------------------------------------------- 1 | package require 2 | 3 | // TestingT is an interface wrapper around *testing.T 4 | type TestingT interface { 5 | Errorf(format string, args ...interface{}) 6 | FailNow() 7 | } 8 | 9 | type tHelper interface { 10 | Helper() 11 | } 12 | 13 | // ComparisonAssertionFunc is a common function prototype when comparing two values. Can be useful 14 | // for table driven tests. 15 | type ComparisonAssertionFunc func(TestingT, interface{}, interface{}, ...interface{}) 16 | 17 | // ValueAssertionFunc is a common function prototype when validating a single value. Can be useful 18 | // for table driven tests. 19 | type ValueAssertionFunc func(TestingT, interface{}, ...interface{}) 20 | 21 | // BoolAssertionFunc is a common function prototype when validating a bool value. Can be useful 22 | // for table driven tests. 23 | type BoolAssertionFunc func(TestingT, bool, ...interface{}) 24 | 25 | // ErrorAssertionFunc is a common function prototype when validating an error value. Can be useful 26 | // for table driven tests. 27 | type ErrorAssertionFunc func(TestingT, error, ...interface{}) 28 | 29 | //go:generate sh -c "cd ../_codegen && go build && cd - && ../_codegen/_codegen -output-package=require -template=require.go.tmpl -include-format-funcs" 30 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/suite/doc.go: -------------------------------------------------------------------------------- 1 | // Package suite contains logic for creating testing suite structs 2 | // and running the methods on those structs as tests. The most useful 3 | // piece of this package is that you can create setup/teardown methods 4 | // on your testing suites, which will run before/after the whole suite 5 | // or individual tests (depending on which interface(s) you 6 | // implement). 7 | // 8 | // A testing suite is usually built by first extending the built-in 9 | // suite functionality from suite.Suite in testify. Alternatively, 10 | // you could reproduce that logic on your own if you wanted (you 11 | // just need to implement the TestingSuite interface from 12 | // suite/interfaces.go). 13 | // 14 | // After that, you can implement any of the interfaces in 15 | // suite/interfaces.go to add setup/teardown functionality to your 16 | // suite, and add any methods that start with "Test" to add tests. 17 | // Methods that do not match any suite interfaces and do not begin 18 | // with "Test" will not be run by testify, and can safely be used as 19 | // helper methods. 20 | // 21 | // Once you've built your testing suite, you need to run the suite 22 | // (using suite.Run from testify) inside any function that matches the 23 | // identity that "go test" is already looking for (i.e. 24 | // func(*testing.T)). 25 | // 26 | // Regular expression to select test suites specified command-line 27 | // argument "-run". Regular expression to select the methods 28 | // of test suites specified command-line argument "-m". 29 | // Suite object has assertion methods. 30 | // 31 | // A crude example: 32 | // // Basic imports 33 | // import ( 34 | // "testing" 35 | // "github.com/stretchr/testify/assert" 36 | // "github.com/stretchr/testify/suite" 37 | // ) 38 | // 39 | // // Define the suite, and absorb the built-in basic suite 40 | // // functionality from testify - including a T() method which 41 | // // returns the current testing context 42 | // type ExampleTestSuite struct { 43 | // suite.Suite 44 | // VariableThatShouldStartAtFive int 45 | // } 46 | // 47 | // // Make sure that VariableThatShouldStartAtFive is set to five 48 | // // before each test 49 | // func (suite *ExampleTestSuite) SetupTest() { 50 | // suite.VariableThatShouldStartAtFive = 5 51 | // } 52 | // 53 | // // All methods that begin with "Test" are run as tests within a 54 | // // suite. 55 | // func (suite *ExampleTestSuite) TestExample() { 56 | // assert.Equal(suite.T(), 5, suite.VariableThatShouldStartAtFive) 57 | // suite.Equal(5, suite.VariableThatShouldStartAtFive) 58 | // } 59 | // 60 | // // In order for 'go test' to run this suite, we need to create 61 | // // a normal test function and pass our suite to suite.Run 62 | // func TestExampleTestSuite(t *testing.T) { 63 | // suite.Run(t, new(ExampleTestSuite)) 64 | // } 65 | package suite 66 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/suite/interfaces.go: -------------------------------------------------------------------------------- 1 | package suite 2 | 3 | import "testing" 4 | 5 | // TestingSuite can store and return the current *testing.T context 6 | // generated by 'go test'. 7 | type TestingSuite interface { 8 | T() *testing.T 9 | SetT(*testing.T) 10 | } 11 | 12 | // SetupAllSuite has a SetupSuite method, which will run before the 13 | // tests in the suite are run. 14 | type SetupAllSuite interface { 15 | SetupSuite() 16 | } 17 | 18 | // SetupTestSuite has a SetupTest method, which will run before each 19 | // test in the suite. 20 | type SetupTestSuite interface { 21 | SetupTest() 22 | } 23 | 24 | // TearDownAllSuite has a TearDownSuite method, which will run after 25 | // all the tests in the suite have been run. 26 | type TearDownAllSuite interface { 27 | TearDownSuite() 28 | } 29 | 30 | // TearDownTestSuite has a TearDownTest method, which will run after 31 | // each test in the suite. 32 | type TearDownTestSuite interface { 33 | TearDownTest() 34 | } 35 | 36 | // BeforeTest has a function to be executed right before the test 37 | // starts and receives the suite and test names as input 38 | type BeforeTest interface { 39 | BeforeTest(suiteName, testName string) 40 | } 41 | 42 | // AfterTest has a function to be executed right after the test 43 | // finishes and receives the suite and test names as input 44 | type AfterTest interface { 45 | AfterTest(suiteName, testName string) 46 | } 47 | -------------------------------------------------------------------------------- /vendor/github.com/stretchr/testify/suite/suite.go: -------------------------------------------------------------------------------- 1 | package suite 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "reflect" 8 | "regexp" 9 | "runtime/debug" 10 | "sync" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | var allTestsFilter = func(_, _ string) (bool, error) { return true, nil } 18 | var matchMethod = flag.String("testify.m", "", "regular expression to select tests of the testify suite to run") 19 | 20 | // Suite is a basic testing suite with methods for storing and 21 | // retrieving the current *testing.T context. 22 | type Suite struct { 23 | *assert.Assertions 24 | require *require.Assertions 25 | t *testing.T 26 | } 27 | 28 | // T retrieves the current *testing.T context. 29 | func (suite *Suite) T() *testing.T { 30 | return suite.t 31 | } 32 | 33 | // SetT sets the current *testing.T context. 34 | func (suite *Suite) SetT(t *testing.T) { 35 | suite.t = t 36 | suite.Assertions = assert.New(t) 37 | suite.require = require.New(t) 38 | } 39 | 40 | // Require returns a require context for suite. 41 | func (suite *Suite) Require() *require.Assertions { 42 | if suite.require == nil { 43 | suite.require = require.New(suite.T()) 44 | } 45 | return suite.require 46 | } 47 | 48 | // Assert returns an assert context for suite. Normally, you can call 49 | // `suite.NoError(expected, actual)`, but for situations where the embedded 50 | // methods are overridden (for example, you might want to override 51 | // assert.Assertions with require.Assertions), this method is provided so you 52 | // can call `suite.Assert().NoError()`. 53 | func (suite *Suite) Assert() *assert.Assertions { 54 | if suite.Assertions == nil { 55 | suite.Assertions = assert.New(suite.T()) 56 | } 57 | return suite.Assertions 58 | } 59 | 60 | func failOnPanic(t *testing.T) { 61 | r := recover() 62 | if r != nil { 63 | t.Errorf("test panicked: %v\n%s", r, debug.Stack()) 64 | t.FailNow() 65 | } 66 | } 67 | 68 | // Run provides suite functionality around golang subtests. It should be 69 | // called in place of t.Run(name, func(t *testing.T)) in test suite code. 70 | // The passed-in func will be executed as a subtest with a fresh instance of t. 71 | // Provides compatibility with go test pkg -run TestSuite/TestName/SubTestName. 72 | func (suite *Suite) Run(name string, subtest func()) bool { 73 | oldT := suite.T() 74 | defer suite.SetT(oldT) 75 | return oldT.Run(name, func(t *testing.T) { 76 | suite.SetT(t) 77 | subtest() 78 | }) 79 | } 80 | 81 | // Run takes a testing suite and runs all of the tests attached 82 | // to it. 83 | func Run(t *testing.T, suite TestingSuite) { 84 | testsSync := &sync.WaitGroup{} 85 | suite.SetT(t) 86 | defer failOnPanic(t) 87 | 88 | suiteSetupDone := false 89 | 90 | methodFinder := reflect.TypeOf(suite) 91 | tests := []testing.InternalTest{} 92 | for index := 0; index < methodFinder.NumMethod(); index++ { 93 | method := methodFinder.Method(index) 94 | ok, err := methodFilter(method.Name) 95 | if err != nil { 96 | fmt.Fprintf(os.Stderr, "testify: invalid regexp for -m: %s\n", err) 97 | os.Exit(1) 98 | } 99 | if !ok { 100 | continue 101 | } 102 | if !suiteSetupDone { 103 | if setupAllSuite, ok := suite.(SetupAllSuite); ok { 104 | setupAllSuite.SetupSuite() 105 | } 106 | defer func() { 107 | if tearDownAllSuite, ok := suite.(TearDownAllSuite); ok { 108 | testsSync.Wait() 109 | tearDownAllSuite.TearDownSuite() 110 | } 111 | }() 112 | suiteSetupDone = true 113 | } 114 | test := testing.InternalTest{ 115 | Name: method.Name, 116 | F: func(t *testing.T) { 117 | defer testsSync.Done() 118 | parentT := suite.T() 119 | suite.SetT(t) 120 | defer failOnPanic(t) 121 | 122 | if setupTestSuite, ok := suite.(SetupTestSuite); ok { 123 | setupTestSuite.SetupTest() 124 | } 125 | if beforeTestSuite, ok := suite.(BeforeTest); ok { 126 | beforeTestSuite.BeforeTest(methodFinder.Elem().Name(), method.Name) 127 | } 128 | defer func() { 129 | if afterTestSuite, ok := suite.(AfterTest); ok { 130 | afterTestSuite.AfterTest(methodFinder.Elem().Name(), method.Name) 131 | } 132 | if tearDownTestSuite, ok := suite.(TearDownTestSuite); ok { 133 | tearDownTestSuite.TearDownTest() 134 | } 135 | suite.SetT(parentT) 136 | }() 137 | method.Func.Call([]reflect.Value{reflect.ValueOf(suite)}) 138 | }, 139 | } 140 | tests = append(tests, test) 141 | testsSync.Add(1) 142 | } 143 | runTests(t, tests) 144 | } 145 | 146 | func runTests(t testing.TB, tests []testing.InternalTest) { 147 | r, ok := t.(runner) 148 | if !ok { // backwards compatibility with Go 1.6 and below 149 | if !testing.RunTests(allTestsFilter, tests) { 150 | t.Fail() 151 | } 152 | return 153 | } 154 | 155 | for _, test := range tests { 156 | r.Run(test.Name, test.F) 157 | } 158 | } 159 | 160 | // Filtering method according to set regular expression 161 | // specified command-line argument -m 162 | func methodFilter(name string) (bool, error) { 163 | if ok, _ := regexp.MatchString("^Test", name); !ok { 164 | return false, nil 165 | } 166 | return regexp.MatchString(*matchMethod, name) 167 | } 168 | 169 | type runner interface { 170 | Run(name string, f func(t *testing.T)) bool 171 | } 172 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2012 Suryandaru Triandana 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 15 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 16 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 17 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 18 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 19 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 20 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 24 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/cache/lru.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package cache 8 | 9 | import ( 10 | "sync" 11 | "unsafe" 12 | ) 13 | 14 | type lruNode struct { 15 | n *Node 16 | h *Handle 17 | ban bool 18 | 19 | next, prev *lruNode 20 | } 21 | 22 | func (n *lruNode) insert(at *lruNode) { 23 | x := at.next 24 | at.next = n 25 | n.prev = at 26 | n.next = x 27 | x.prev = n 28 | } 29 | 30 | func (n *lruNode) remove() { 31 | if n.prev != nil { 32 | n.prev.next = n.next 33 | n.next.prev = n.prev 34 | n.prev = nil 35 | n.next = nil 36 | } else { 37 | panic("BUG: removing removed node") 38 | } 39 | } 40 | 41 | type lru struct { 42 | mu sync.Mutex 43 | capacity int 44 | used int 45 | recent lruNode 46 | } 47 | 48 | func (r *lru) reset() { 49 | r.recent.next = &r.recent 50 | r.recent.prev = &r.recent 51 | r.used = 0 52 | } 53 | 54 | func (r *lru) Capacity() int { 55 | r.mu.Lock() 56 | defer r.mu.Unlock() 57 | return r.capacity 58 | } 59 | 60 | func (r *lru) SetCapacity(capacity int) { 61 | var evicted []*lruNode 62 | 63 | r.mu.Lock() 64 | r.capacity = capacity 65 | for r.used > r.capacity { 66 | rn := r.recent.prev 67 | if rn == nil { 68 | panic("BUG: invalid LRU used or capacity counter") 69 | } 70 | rn.remove() 71 | rn.n.CacheData = nil 72 | r.used -= rn.n.Size() 73 | evicted = append(evicted, rn) 74 | } 75 | r.mu.Unlock() 76 | 77 | for _, rn := range evicted { 78 | rn.h.Release() 79 | } 80 | } 81 | 82 | func (r *lru) Promote(n *Node) { 83 | var evicted []*lruNode 84 | 85 | r.mu.Lock() 86 | if n.CacheData == nil { 87 | if n.Size() <= r.capacity { 88 | rn := &lruNode{n: n, h: n.GetHandle()} 89 | rn.insert(&r.recent) 90 | n.CacheData = unsafe.Pointer(rn) 91 | r.used += n.Size() 92 | 93 | for r.used > r.capacity { 94 | rn := r.recent.prev 95 | if rn == nil { 96 | panic("BUG: invalid LRU used or capacity counter") 97 | } 98 | rn.remove() 99 | rn.n.CacheData = nil 100 | r.used -= rn.n.Size() 101 | evicted = append(evicted, rn) 102 | } 103 | } 104 | } else { 105 | rn := (*lruNode)(n.CacheData) 106 | if !rn.ban { 107 | rn.remove() 108 | rn.insert(&r.recent) 109 | } 110 | } 111 | r.mu.Unlock() 112 | 113 | for _, rn := range evicted { 114 | rn.h.Release() 115 | } 116 | } 117 | 118 | func (r *lru) Ban(n *Node) { 119 | r.mu.Lock() 120 | if n.CacheData == nil { 121 | n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true}) 122 | } else { 123 | rn := (*lruNode)(n.CacheData) 124 | if !rn.ban { 125 | rn.remove() 126 | rn.ban = true 127 | r.used -= rn.n.Size() 128 | r.mu.Unlock() 129 | 130 | rn.h.Release() 131 | rn.h = nil 132 | return 133 | } 134 | } 135 | r.mu.Unlock() 136 | } 137 | 138 | func (r *lru) Evict(n *Node) { 139 | r.mu.Lock() 140 | rn := (*lruNode)(n.CacheData) 141 | if rn == nil || rn.ban { 142 | r.mu.Unlock() 143 | return 144 | } 145 | n.CacheData = nil 146 | r.mu.Unlock() 147 | 148 | rn.h.Release() 149 | } 150 | 151 | func (r *lru) EvictNS(ns uint64) { 152 | var evicted []*lruNode 153 | 154 | r.mu.Lock() 155 | for e := r.recent.prev; e != &r.recent; { 156 | rn := e 157 | e = e.prev 158 | if rn.n.NS() == ns { 159 | rn.remove() 160 | rn.n.CacheData = nil 161 | r.used -= rn.n.Size() 162 | evicted = append(evicted, rn) 163 | } 164 | } 165 | r.mu.Unlock() 166 | 167 | for _, rn := range evicted { 168 | rn.h.Release() 169 | } 170 | } 171 | 172 | func (r *lru) EvictAll() { 173 | r.mu.Lock() 174 | back := r.recent.prev 175 | for rn := back; rn != &r.recent; rn = rn.prev { 176 | rn.n.CacheData = nil 177 | } 178 | r.reset() 179 | r.mu.Unlock() 180 | 181 | for rn := back; rn != &r.recent; rn = rn.prev { 182 | rn.h.Release() 183 | } 184 | } 185 | 186 | func (r *lru) Close() error { 187 | return nil 188 | } 189 | 190 | // NewLRU create a new LRU-cache. 191 | func NewLRU(capacity int) Cacher { 192 | r := &lru{capacity: capacity} 193 | r.reset() 194 | return r 195 | } 196 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/comparer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/comparer" 11 | ) 12 | 13 | type iComparer struct { 14 | ucmp comparer.Comparer 15 | } 16 | 17 | func (icmp *iComparer) uName() string { 18 | return icmp.ucmp.Name() 19 | } 20 | 21 | func (icmp *iComparer) uCompare(a, b []byte) int { 22 | return icmp.ucmp.Compare(a, b) 23 | } 24 | 25 | func (icmp *iComparer) uSeparator(dst, a, b []byte) []byte { 26 | return icmp.ucmp.Separator(dst, a, b) 27 | } 28 | 29 | func (icmp *iComparer) uSuccessor(dst, b []byte) []byte { 30 | return icmp.ucmp.Successor(dst, b) 31 | } 32 | 33 | func (icmp *iComparer) Name() string { 34 | return icmp.uName() 35 | } 36 | 37 | func (icmp *iComparer) Compare(a, b []byte) int { 38 | x := icmp.uCompare(internalKey(a).ukey(), internalKey(b).ukey()) 39 | if x == 0 { 40 | if m, n := internalKey(a).num(), internalKey(b).num(); m > n { 41 | return -1 42 | } else if m < n { 43 | return 1 44 | } 45 | } 46 | return x 47 | } 48 | 49 | func (icmp *iComparer) Separator(dst, a, b []byte) []byte { 50 | ua, ub := internalKey(a).ukey(), internalKey(b).ukey() 51 | dst = icmp.uSeparator(dst, ua, ub) 52 | if dst != nil && len(dst) < len(ua) && icmp.uCompare(ua, dst) < 0 { 53 | // Append earliest possible number. 54 | return append(dst, keyMaxNumBytes...) 55 | } 56 | return nil 57 | } 58 | 59 | func (icmp *iComparer) Successor(dst, b []byte) []byte { 60 | ub := internalKey(b).ukey() 61 | dst = icmp.uSuccessor(dst, ub) 62 | if dst != nil && len(dst) < len(ub) && icmp.uCompare(ub, dst) < 0 { 63 | // Append earliest possible number. 64 | return append(dst, keyMaxNumBytes...) 65 | } 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package comparer 8 | 9 | import "bytes" 10 | 11 | type bytesComparer struct{} 12 | 13 | func (bytesComparer) Compare(a, b []byte) int { 14 | return bytes.Compare(a, b) 15 | } 16 | 17 | func (bytesComparer) Name() string { 18 | return "leveldb.BytewiseComparator" 19 | } 20 | 21 | func (bytesComparer) Separator(dst, a, b []byte) []byte { 22 | i, n := 0, len(a) 23 | if n > len(b) { 24 | n = len(b) 25 | } 26 | for ; i < n && a[i] == b[i]; i++ { 27 | } 28 | if i >= n { 29 | // Do not shorten if one string is a prefix of the other 30 | } else if c := a[i]; c < 0xff && c+1 < b[i] { 31 | dst = append(dst, a[:i+1]...) 32 | dst[len(dst)-1]++ 33 | return dst 34 | } 35 | return nil 36 | } 37 | 38 | func (bytesComparer) Successor(dst, b []byte) []byte { 39 | for i, c := range b { 40 | if c != 0xff { 41 | dst = append(dst, b[:i+1]...) 42 | dst[len(dst)-1]++ 43 | return dst 44 | } 45 | } 46 | return nil 47 | } 48 | 49 | // DefaultComparer are default implementation of the Comparer interface. 50 | // It uses the natural ordering, consistent with bytes.Compare. 51 | var DefaultComparer = bytesComparer{} 52 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package comparer provides interface and implementation for ordering 8 | // sets of data. 9 | package comparer 10 | 11 | // BasicComparer is the interface that wraps the basic Compare method. 12 | type BasicComparer interface { 13 | // Compare returns -1, 0, or +1 depending on whether a is 'less than', 14 | // 'equal to' or 'greater than' b. The two arguments can only be 'equal' 15 | // if their contents are exactly equal. Furthermore, the empty slice 16 | // must be 'less than' any non-empty slice. 17 | Compare(a, b []byte) int 18 | } 19 | 20 | // Comparer defines a total ordering over the space of []byte keys: a 'less 21 | // than' relationship. 22 | type Comparer interface { 23 | BasicComparer 24 | 25 | // Name returns name of the comparer. 26 | // 27 | // The Level-DB on-disk format stores the comparer name, and opening a 28 | // database with a different comparer from the one it was created with 29 | // will result in an error. 30 | // 31 | // An implementation to a new name whenever the comparer implementation 32 | // changes in a way that will cause the relative ordering of any two keys 33 | // to change. 34 | // 35 | // Names starting with "leveldb." are reserved and should not be used 36 | // by any users of this package. 37 | Name() string 38 | 39 | // Bellow are advanced functions used to reduce the space requirements 40 | // for internal data structures such as index blocks. 41 | 42 | // Separator appends a sequence of bytes x to dst such that a <= x && x < b, 43 | // where 'less than' is consistent with Compare. An implementation should 44 | // return nil if x equal to a. 45 | // 46 | // Either contents of a or b should not by any means modified. Doing so 47 | // may cause corruption on the internal state. 48 | Separator(dst, a, b []byte) []byte 49 | 50 | // Successor appends a sequence of bytes x to dst such that x >= b, where 51 | // 'less than' is consistent with Compare. An implementation should return 52 | // nil if x equal to b. 53 | // 54 | // Contents of b should not by any means modified. Doing so may cause 55 | // corruption on the internal state. 56 | Successor(dst, b []byte) []byte 57 | } 58 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/db_snapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "container/list" 11 | "fmt" 12 | "runtime" 13 | "sync" 14 | "sync/atomic" 15 | 16 | "github.com/syndtr/goleveldb/leveldb/iterator" 17 | "github.com/syndtr/goleveldb/leveldb/opt" 18 | "github.com/syndtr/goleveldb/leveldb/util" 19 | ) 20 | 21 | type snapshotElement struct { 22 | seq uint64 23 | ref int 24 | e *list.Element 25 | } 26 | 27 | // Acquires a snapshot, based on latest sequence. 28 | func (db *DB) acquireSnapshot() *snapshotElement { 29 | db.snapsMu.Lock() 30 | defer db.snapsMu.Unlock() 31 | 32 | seq := db.getSeq() 33 | 34 | if e := db.snapsList.Back(); e != nil { 35 | se := e.Value.(*snapshotElement) 36 | if se.seq == seq { 37 | se.ref++ 38 | return se 39 | } else if seq < se.seq { 40 | panic("leveldb: sequence number is not increasing") 41 | } 42 | } 43 | se := &snapshotElement{seq: seq, ref: 1} 44 | se.e = db.snapsList.PushBack(se) 45 | return se 46 | } 47 | 48 | // Releases given snapshot element. 49 | func (db *DB) releaseSnapshot(se *snapshotElement) { 50 | db.snapsMu.Lock() 51 | defer db.snapsMu.Unlock() 52 | 53 | se.ref-- 54 | if se.ref == 0 { 55 | db.snapsList.Remove(se.e) 56 | se.e = nil 57 | } else if se.ref < 0 { 58 | panic("leveldb: Snapshot: negative element reference") 59 | } 60 | } 61 | 62 | // Gets minimum sequence that not being snapshotted. 63 | func (db *DB) minSeq() uint64 { 64 | db.snapsMu.Lock() 65 | defer db.snapsMu.Unlock() 66 | 67 | if e := db.snapsList.Front(); e != nil { 68 | return e.Value.(*snapshotElement).seq 69 | } 70 | 71 | return db.getSeq() 72 | } 73 | 74 | // Snapshot is a DB snapshot. 75 | type Snapshot struct { 76 | db *DB 77 | elem *snapshotElement 78 | mu sync.RWMutex 79 | released bool 80 | } 81 | 82 | // Creates new snapshot object. 83 | func (db *DB) newSnapshot() *Snapshot { 84 | snap := &Snapshot{ 85 | db: db, 86 | elem: db.acquireSnapshot(), 87 | } 88 | atomic.AddInt32(&db.aliveSnaps, 1) 89 | runtime.SetFinalizer(snap, (*Snapshot).Release) 90 | return snap 91 | } 92 | 93 | func (snap *Snapshot) String() string { 94 | return fmt.Sprintf("leveldb.Snapshot{%d}", snap.elem.seq) 95 | } 96 | 97 | // Get gets the value for the given key. It returns ErrNotFound if 98 | // the DB does not contains the key. 99 | // 100 | // The caller should not modify the contents of the returned slice, but 101 | // it is safe to modify the contents of the argument after Get returns. 102 | func (snap *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { 103 | err = snap.db.ok() 104 | if err != nil { 105 | return 106 | } 107 | snap.mu.RLock() 108 | defer snap.mu.RUnlock() 109 | if snap.released { 110 | err = ErrSnapshotReleased 111 | return 112 | } 113 | return snap.db.get(nil, nil, key, snap.elem.seq, ro) 114 | } 115 | 116 | // Has returns true if the DB does contains the given key. 117 | // 118 | // It is safe to modify the contents of the argument after Get returns. 119 | func (snap *Snapshot) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) { 120 | err = snap.db.ok() 121 | if err != nil { 122 | return 123 | } 124 | snap.mu.RLock() 125 | defer snap.mu.RUnlock() 126 | if snap.released { 127 | err = ErrSnapshotReleased 128 | return 129 | } 130 | return snap.db.has(nil, nil, key, snap.elem.seq, ro) 131 | } 132 | 133 | // NewIterator returns an iterator for the snapshot of the underlying DB. 134 | // The returned iterator is not safe for concurrent use, but it is safe to use 135 | // multiple iterators concurrently, with each in a dedicated goroutine. 136 | // It is also safe to use an iterator concurrently with modifying its 137 | // underlying DB. The resultant key/value pairs are guaranteed to be 138 | // consistent. 139 | // 140 | // Slice allows slicing the iterator to only contains keys in the given 141 | // range. A nil Range.Start is treated as a key before all keys in the 142 | // DB. And a nil Range.Limit is treated as a key after all keys in 143 | // the DB. 144 | // 145 | // WARNING: Any slice returned by interator (e.g. slice returned by calling 146 | // Iterator.Key() or Iterator.Value() methods), its content should not be 147 | // modified unless noted otherwise. 148 | // 149 | // The iterator must be released after use, by calling Release method. 150 | // Releasing the snapshot doesn't mean releasing the iterator too, the 151 | // iterator would be still valid until released. 152 | // 153 | // Also read Iterator documentation of the leveldb/iterator package. 154 | func (snap *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { 155 | if err := snap.db.ok(); err != nil { 156 | return iterator.NewEmptyIterator(err) 157 | } 158 | snap.mu.Lock() 159 | defer snap.mu.Unlock() 160 | if snap.released { 161 | return iterator.NewEmptyIterator(ErrSnapshotReleased) 162 | } 163 | // Since iterator already hold version ref, it doesn't need to 164 | // hold snapshot ref. 165 | return snap.db.newIterator(nil, nil, snap.elem.seq, slice, ro) 166 | } 167 | 168 | // Release releases the snapshot. This will not release any returned 169 | // iterators, the iterators would still be valid until released or the 170 | // underlying DB is closed. 171 | // 172 | // Other methods should not be called after the snapshot has been released. 173 | func (snap *Snapshot) Release() { 174 | snap.mu.Lock() 175 | defer snap.mu.Unlock() 176 | 177 | if !snap.released { 178 | // Clear the finalizer. 179 | runtime.SetFinalizer(snap, nil) 180 | 181 | snap.released = true 182 | snap.db.releaseSnapshot(snap.elem) 183 | atomic.AddInt32(&snap.db.aliveSnaps, -1) 184 | snap.db = nil 185 | snap.elem = nil 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/db_state.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "errors" 11 | "sync/atomic" 12 | "time" 13 | 14 | "github.com/syndtr/goleveldb/leveldb/journal" 15 | "github.com/syndtr/goleveldb/leveldb/memdb" 16 | "github.com/syndtr/goleveldb/leveldb/storage" 17 | ) 18 | 19 | var ( 20 | errHasFrozenMem = errors.New("has frozen mem") 21 | ) 22 | 23 | type memDB struct { 24 | db *DB 25 | *memdb.DB 26 | ref int32 27 | } 28 | 29 | func (m *memDB) getref() int32 { 30 | return atomic.LoadInt32(&m.ref) 31 | } 32 | 33 | func (m *memDB) incref() { 34 | atomic.AddInt32(&m.ref, 1) 35 | } 36 | 37 | func (m *memDB) decref() { 38 | if ref := atomic.AddInt32(&m.ref, -1); ref == 0 { 39 | // Only put back memdb with std capacity. 40 | if m.Capacity() == m.db.s.o.GetWriteBuffer() { 41 | m.Reset() 42 | m.db.mpoolPut(m.DB) 43 | } 44 | m.db = nil 45 | m.DB = nil 46 | } else if ref < 0 { 47 | panic("negative memdb ref") 48 | } 49 | } 50 | 51 | // Get latest sequence number. 52 | func (db *DB) getSeq() uint64 { 53 | return atomic.LoadUint64(&db.seq) 54 | } 55 | 56 | // Atomically adds delta to seq. 57 | func (db *DB) addSeq(delta uint64) { 58 | atomic.AddUint64(&db.seq, delta) 59 | } 60 | 61 | func (db *DB) setSeq(seq uint64) { 62 | atomic.StoreUint64(&db.seq, seq) 63 | } 64 | 65 | func (db *DB) sampleSeek(ikey internalKey) { 66 | v := db.s.version() 67 | if v.sampleSeek(ikey) { 68 | // Trigger table compaction. 69 | db.compTrigger(db.tcompCmdC) 70 | } 71 | v.release() 72 | } 73 | 74 | func (db *DB) mpoolPut(mem *memdb.DB) { 75 | if !db.isClosed() { 76 | select { 77 | case db.memPool <- mem: 78 | default: 79 | } 80 | } 81 | } 82 | 83 | func (db *DB) mpoolGet(n int) *memDB { 84 | var mdb *memdb.DB 85 | select { 86 | case mdb = <-db.memPool: 87 | default: 88 | } 89 | if mdb == nil || mdb.Capacity() < n { 90 | mdb = memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n)) 91 | } 92 | return &memDB{ 93 | db: db, 94 | DB: mdb, 95 | } 96 | } 97 | 98 | func (db *DB) mpoolDrain() { 99 | ticker := time.NewTicker(30 * time.Second) 100 | for { 101 | select { 102 | case <-ticker.C: 103 | select { 104 | case <-db.memPool: 105 | default: 106 | } 107 | case <-db.closeC: 108 | ticker.Stop() 109 | // Make sure the pool is drained. 110 | select { 111 | case <-db.memPool: 112 | case <-time.After(time.Second): 113 | } 114 | close(db.memPool) 115 | return 116 | } 117 | } 118 | } 119 | 120 | // Create new memdb and froze the old one; need external synchronization. 121 | // newMem only called synchronously by the writer. 122 | func (db *DB) newMem(n int) (mem *memDB, err error) { 123 | fd := storage.FileDesc{Type: storage.TypeJournal, Num: db.s.allocFileNum()} 124 | w, err := db.s.stor.Create(fd) 125 | if err != nil { 126 | db.s.reuseFileNum(fd.Num) 127 | return 128 | } 129 | 130 | db.memMu.Lock() 131 | defer db.memMu.Unlock() 132 | 133 | if db.frozenMem != nil { 134 | return nil, errHasFrozenMem 135 | } 136 | 137 | if db.journal == nil { 138 | db.journal = journal.NewWriter(w) 139 | } else { 140 | db.journal.Reset(w) 141 | db.journalWriter.Close() 142 | db.frozenJournalFd = db.journalFd 143 | } 144 | db.journalWriter = w 145 | db.journalFd = fd 146 | db.frozenMem = db.mem 147 | mem = db.mpoolGet(n) 148 | mem.incref() // for self 149 | mem.incref() // for caller 150 | db.mem = mem 151 | // The seq only incremented by the writer. And whoever called newMem 152 | // should hold write lock, so no need additional synchronization here. 153 | db.frozenSeq = db.seq 154 | return 155 | } 156 | 157 | // Get all memdbs. 158 | func (db *DB) getMems() (e, f *memDB) { 159 | db.memMu.RLock() 160 | defer db.memMu.RUnlock() 161 | if db.mem != nil { 162 | db.mem.incref() 163 | } else if !db.isClosed() { 164 | panic("nil effective mem") 165 | } 166 | if db.frozenMem != nil { 167 | db.frozenMem.incref() 168 | } 169 | return db.mem, db.frozenMem 170 | } 171 | 172 | // Get effective memdb. 173 | func (db *DB) getEffectiveMem() *memDB { 174 | db.memMu.RLock() 175 | defer db.memMu.RUnlock() 176 | if db.mem != nil { 177 | db.mem.incref() 178 | } else if !db.isClosed() { 179 | panic("nil effective mem") 180 | } 181 | return db.mem 182 | } 183 | 184 | // Check whether we has frozen memdb. 185 | func (db *DB) hasFrozenMem() bool { 186 | db.memMu.RLock() 187 | defer db.memMu.RUnlock() 188 | return db.frozenMem != nil 189 | } 190 | 191 | // Get frozen memdb. 192 | func (db *DB) getFrozenMem() *memDB { 193 | db.memMu.RLock() 194 | defer db.memMu.RUnlock() 195 | if db.frozenMem != nil { 196 | db.frozenMem.incref() 197 | } 198 | return db.frozenMem 199 | } 200 | 201 | // Drop frozen memdb; assume that frozen memdb isn't nil. 202 | func (db *DB) dropFrozenMem() { 203 | db.memMu.Lock() 204 | if err := db.s.stor.Remove(db.frozenJournalFd); err != nil { 205 | db.logf("journal@remove removing @%d %q", db.frozenJournalFd.Num, err) 206 | } else { 207 | db.logf("journal@remove removed @%d", db.frozenJournalFd.Num) 208 | } 209 | db.frozenJournalFd = storage.FileDesc{} 210 | db.frozenMem.decref() 211 | db.frozenMem = nil 212 | db.memMu.Unlock() 213 | } 214 | 215 | // Clear mems ptr; used by DB.Close(). 216 | func (db *DB) clearMems() { 217 | db.memMu.Lock() 218 | db.mem = nil 219 | db.frozenMem = nil 220 | db.memMu.Unlock() 221 | } 222 | 223 | // Set closed flag; return true if not already closed. 224 | func (db *DB) setClosed() bool { 225 | return atomic.CompareAndSwapUint32(&db.closed, 0, 1) 226 | } 227 | 228 | // Check whether DB was closed. 229 | func (db *DB) isClosed() bool { 230 | return atomic.LoadUint32(&db.closed) != 0 231 | } 232 | 233 | // Check read ok status. 234 | func (db *DB) ok() error { 235 | if db.isClosed() { 236 | return ErrClosed 237 | } 238 | return nil 239 | } 240 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/db_util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/errors" 11 | "github.com/syndtr/goleveldb/leveldb/iterator" 12 | "github.com/syndtr/goleveldb/leveldb/opt" 13 | "github.com/syndtr/goleveldb/leveldb/storage" 14 | "github.com/syndtr/goleveldb/leveldb/util" 15 | ) 16 | 17 | // Reader is the interface that wraps basic Get and NewIterator methods. 18 | // This interface implemented by both DB and Snapshot. 19 | type Reader interface { 20 | Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) 21 | NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator 22 | } 23 | 24 | // Sizes is list of size. 25 | type Sizes []int64 26 | 27 | // Sum returns sum of the sizes. 28 | func (sizes Sizes) Sum() int64 { 29 | var sum int64 30 | for _, size := range sizes { 31 | sum += size 32 | } 33 | return sum 34 | } 35 | 36 | // Logging. 37 | func (db *DB) log(v ...interface{}) { db.s.log(v...) } 38 | func (db *DB) logf(format string, v ...interface{}) { db.s.logf(format, v...) } 39 | 40 | // Check and clean files. 41 | func (db *DB) checkAndCleanFiles() error { 42 | v := db.s.version() 43 | defer v.release() 44 | 45 | tmap := make(map[int64]bool) 46 | for _, tables := range v.levels { 47 | for _, t := range tables { 48 | tmap[t.fd.Num] = false 49 | } 50 | } 51 | 52 | fds, err := db.s.stor.List(storage.TypeAll) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | var nt int 58 | var rem []storage.FileDesc 59 | for _, fd := range fds { 60 | keep := true 61 | switch fd.Type { 62 | case storage.TypeManifest: 63 | keep = fd.Num >= db.s.manifestFd.Num 64 | case storage.TypeJournal: 65 | if !db.frozenJournalFd.Zero() { 66 | keep = fd.Num >= db.frozenJournalFd.Num 67 | } else { 68 | keep = fd.Num >= db.journalFd.Num 69 | } 70 | case storage.TypeTable: 71 | _, keep = tmap[fd.Num] 72 | if keep { 73 | tmap[fd.Num] = true 74 | nt++ 75 | } 76 | } 77 | 78 | if !keep { 79 | rem = append(rem, fd) 80 | } 81 | } 82 | 83 | if nt != len(tmap) { 84 | var mfds []storage.FileDesc 85 | for num, present := range tmap { 86 | if !present { 87 | mfds = append(mfds, storage.FileDesc{Type: storage.TypeTable, Num: num}) 88 | db.logf("db@janitor table missing @%d", num) 89 | } 90 | } 91 | return errors.NewErrCorrupted(storage.FileDesc{}, &errors.ErrMissingFiles{Fds: mfds}) 92 | } 93 | 94 | db.logf("db@janitor F·%d G·%d", len(fds), len(rem)) 95 | for _, fd := range rem { 96 | db.logf("db@janitor removing %s-%d", fd.Type, fd.Num) 97 | if err := db.s.stor.Remove(fd); err != nil { 98 | return err 99 | } 100 | } 101 | return nil 102 | } 103 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package leveldb provides implementation of LevelDB key/value database. 8 | // 9 | // Create or open a database: 10 | // 11 | // // The returned DB instance is safe for concurrent use. Which mean that all 12 | // // DB's methods may be called concurrently from multiple goroutine. 13 | // db, err := leveldb.OpenFile("path/to/db", nil) 14 | // ... 15 | // defer db.Close() 16 | // ... 17 | // 18 | // Read or modify the database content: 19 | // 20 | // // Remember that the contents of the returned slice should not be modified. 21 | // data, err := db.Get([]byte("key"), nil) 22 | // ... 23 | // err = db.Put([]byte("key"), []byte("value"), nil) 24 | // ... 25 | // err = db.Delete([]byte("key"), nil) 26 | // ... 27 | // 28 | // Iterate over database content: 29 | // 30 | // iter := db.NewIterator(nil, nil) 31 | // for iter.Next() { 32 | // // Remember that the contents of the returned slice should not be modified, and 33 | // // only valid until the next call to Next. 34 | // key := iter.Key() 35 | // value := iter.Value() 36 | // ... 37 | // } 38 | // iter.Release() 39 | // err = iter.Error() 40 | // ... 41 | // 42 | // Iterate over subset of database content with a particular prefix: 43 | // iter := db.NewIterator(util.BytesPrefix([]byte("foo-")), nil) 44 | // for iter.Next() { 45 | // // Use key/value. 46 | // ... 47 | // } 48 | // iter.Release() 49 | // err = iter.Error() 50 | // ... 51 | // 52 | // Seek-then-Iterate: 53 | // 54 | // iter := db.NewIterator(nil, nil) 55 | // for ok := iter.Seek(key); ok; ok = iter.Next() { 56 | // // Use key/value. 57 | // ... 58 | // } 59 | // iter.Release() 60 | // err = iter.Error() 61 | // ... 62 | // 63 | // Iterate over subset of database content: 64 | // 65 | // iter := db.NewIterator(&util.Range{Start: []byte("foo"), Limit: []byte("xoo")}, nil) 66 | // for iter.Next() { 67 | // // Use key/value. 68 | // ... 69 | // } 70 | // iter.Release() 71 | // err = iter.Error() 72 | // ... 73 | // 74 | // Batch writes: 75 | // 76 | // batch := new(leveldb.Batch) 77 | // batch.Put([]byte("foo"), []byte("value")) 78 | // batch.Put([]byte("bar"), []byte("another value")) 79 | // batch.Delete([]byte("baz")) 80 | // err = db.Write(batch, nil) 81 | // ... 82 | // 83 | // Use bloom filter: 84 | // 85 | // o := &opt.Options{ 86 | // Filter: filter.NewBloomFilter(10), 87 | // } 88 | // db, err := leveldb.OpenFile("path/to/db", o) 89 | // ... 90 | // defer db.Close() 91 | // ... 92 | package leveldb 93 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/errors" 11 | ) 12 | 13 | // Common errors. 14 | var ( 15 | ErrNotFound = errors.ErrNotFound 16 | ErrReadOnly = errors.New("leveldb: read-only mode") 17 | ErrSnapshotReleased = errors.New("leveldb: snapshot released") 18 | ErrIterReleased = errors.New("leveldb: iterator released") 19 | ErrClosed = errors.New("leveldb: closed") 20 | ) 21 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/errors/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package errors provides common error types used throughout leveldb. 8 | package errors 9 | 10 | import ( 11 | "errors" 12 | "fmt" 13 | 14 | "github.com/syndtr/goleveldb/leveldb/storage" 15 | "github.com/syndtr/goleveldb/leveldb/util" 16 | ) 17 | 18 | // Common errors. 19 | var ( 20 | ErrNotFound = New("leveldb: not found") 21 | ErrReleased = util.ErrReleased 22 | ErrHasReleaser = util.ErrHasReleaser 23 | ) 24 | 25 | // New returns an error that formats as the given text. 26 | func New(text string) error { 27 | return errors.New(text) 28 | } 29 | 30 | // ErrCorrupted is the type that wraps errors that indicate corruption in 31 | // the database. 32 | type ErrCorrupted struct { 33 | Fd storage.FileDesc 34 | Err error 35 | } 36 | 37 | func (e *ErrCorrupted) Error() string { 38 | if !e.Fd.Zero() { 39 | return fmt.Sprintf("%v [file=%v]", e.Err, e.Fd) 40 | } 41 | return e.Err.Error() 42 | } 43 | 44 | // NewErrCorrupted creates new ErrCorrupted error. 45 | func NewErrCorrupted(fd storage.FileDesc, err error) error { 46 | return &ErrCorrupted{fd, err} 47 | } 48 | 49 | // IsCorrupted returns a boolean indicating whether the error is indicating 50 | // a corruption. 51 | func IsCorrupted(err error) bool { 52 | switch err.(type) { 53 | case *ErrCorrupted: 54 | return true 55 | case *storage.ErrCorrupted: 56 | return true 57 | } 58 | return false 59 | } 60 | 61 | // ErrMissingFiles is the type that indicating a corruption due to missing 62 | // files. ErrMissingFiles always wrapped with ErrCorrupted. 63 | type ErrMissingFiles struct { 64 | Fds []storage.FileDesc 65 | } 66 | 67 | func (e *ErrMissingFiles) Error() string { return "file missing" } 68 | 69 | // SetFd sets 'file info' of the given error with the given file. 70 | // Currently only ErrCorrupted is supported, otherwise will do nothing. 71 | func SetFd(err error, fd storage.FileDesc) error { 72 | switch x := err.(type) { 73 | case *ErrCorrupted: 74 | x.Fd = fd 75 | return x 76 | } 77 | return err 78 | } 79 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/filter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/filter" 11 | ) 12 | 13 | type iFilter struct { 14 | filter.Filter 15 | } 16 | 17 | func (f iFilter) Contains(filter, key []byte) bool { 18 | return f.Filter.Contains(filter, internalKey(key).ukey()) 19 | } 20 | 21 | func (f iFilter) NewGenerator() filter.FilterGenerator { 22 | return iFilterGenerator{f.Filter.NewGenerator()} 23 | } 24 | 25 | type iFilterGenerator struct { 26 | filter.FilterGenerator 27 | } 28 | 29 | func (g iFilterGenerator) Add(key []byte) { 30 | g.FilterGenerator.Add(internalKey(key).ukey()) 31 | } 32 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/filter/bloom.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package filter 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/util" 11 | ) 12 | 13 | func bloomHash(key []byte) uint32 { 14 | return util.Hash(key, 0xbc9f1d34) 15 | } 16 | 17 | type bloomFilter int 18 | 19 | // The bloom filter serializes its parameters and is backward compatible 20 | // with respect to them. Therefor, its parameters are not added to its 21 | // name. 22 | func (bloomFilter) Name() string { 23 | return "leveldb.BuiltinBloomFilter" 24 | } 25 | 26 | func (f bloomFilter) Contains(filter, key []byte) bool { 27 | nBytes := len(filter) - 1 28 | if nBytes < 1 { 29 | return false 30 | } 31 | nBits := uint32(nBytes * 8) 32 | 33 | // Use the encoded k so that we can read filters generated by 34 | // bloom filters created using different parameters. 35 | k := filter[nBytes] 36 | if k > 30 { 37 | // Reserved for potentially new encodings for short bloom filters. 38 | // Consider it a match. 39 | return true 40 | } 41 | 42 | kh := bloomHash(key) 43 | delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits 44 | for j := uint8(0); j < k; j++ { 45 | bitpos := kh % nBits 46 | if (uint32(filter[bitpos/8]) & (1 << (bitpos % 8))) == 0 { 47 | return false 48 | } 49 | kh += delta 50 | } 51 | return true 52 | } 53 | 54 | func (f bloomFilter) NewGenerator() FilterGenerator { 55 | // Round down to reduce probing cost a little bit. 56 | k := uint8(f * 69 / 100) // 0.69 =~ ln(2) 57 | if k < 1 { 58 | k = 1 59 | } else if k > 30 { 60 | k = 30 61 | } 62 | return &bloomFilterGenerator{ 63 | n: int(f), 64 | k: k, 65 | } 66 | } 67 | 68 | type bloomFilterGenerator struct { 69 | n int 70 | k uint8 71 | 72 | keyHashes []uint32 73 | } 74 | 75 | func (g *bloomFilterGenerator) Add(key []byte) { 76 | // Use double-hashing to generate a sequence of hash values. 77 | // See analysis in [Kirsch,Mitzenmacher 2006]. 78 | g.keyHashes = append(g.keyHashes, bloomHash(key)) 79 | } 80 | 81 | func (g *bloomFilterGenerator) Generate(b Buffer) { 82 | // Compute bloom filter size (in both bits and bytes) 83 | nBits := uint32(len(g.keyHashes) * g.n) 84 | // For small n, we can see a very high false positive rate. Fix it 85 | // by enforcing a minimum bloom filter length. 86 | if nBits < 64 { 87 | nBits = 64 88 | } 89 | nBytes := (nBits + 7) / 8 90 | nBits = nBytes * 8 91 | 92 | dest := b.Alloc(int(nBytes) + 1) 93 | dest[nBytes] = g.k 94 | for _, kh := range g.keyHashes { 95 | delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits 96 | for j := uint8(0); j < g.k; j++ { 97 | bitpos := kh % nBits 98 | dest[bitpos/8] |= (1 << (bitpos % 8)) 99 | kh += delta 100 | } 101 | } 102 | 103 | g.keyHashes = g.keyHashes[:0] 104 | } 105 | 106 | // NewBloomFilter creates a new initialized bloom filter for given 107 | // bitsPerKey. 108 | // 109 | // Since bitsPerKey is persisted individually for each bloom filter 110 | // serialization, bloom filters are backwards compatible with respect to 111 | // changing bitsPerKey. This means that no big performance penalty will 112 | // be experienced when changing the parameter. See documentation for 113 | // opt.Options.Filter for more information. 114 | func NewBloomFilter(bitsPerKey int) Filter { 115 | return bloomFilter(bitsPerKey) 116 | } 117 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/filter/filter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package filter provides interface and implementation of probabilistic 8 | // data structure. 9 | // 10 | // The filter is resposible for creating small filter from a set of keys. 11 | // These filter will then used to test whether a key is a member of the set. 12 | // In many cases, a filter can cut down the number of disk seeks from a 13 | // handful to a single disk seek per DB.Get call. 14 | package filter 15 | 16 | // Buffer is the interface that wraps basic Alloc, Write and WriteByte methods. 17 | type Buffer interface { 18 | // Alloc allocs n bytes of slice from the buffer. This also advancing 19 | // write offset. 20 | Alloc(n int) []byte 21 | 22 | // Write appends the contents of p to the buffer. 23 | Write(p []byte) (n int, err error) 24 | 25 | // WriteByte appends the byte c to the buffer. 26 | WriteByte(c byte) error 27 | } 28 | 29 | // Filter is the filter. 30 | type Filter interface { 31 | // Name returns the name of this policy. 32 | // 33 | // Note that if the filter encoding changes in an incompatible way, 34 | // the name returned by this method must be changed. Otherwise, old 35 | // incompatible filters may be passed to methods of this type. 36 | Name() string 37 | 38 | // NewGenerator creates a new filter generator. 39 | NewGenerator() FilterGenerator 40 | 41 | // Contains returns true if the filter contains the given key. 42 | // 43 | // The filter are filters generated by the filter generator. 44 | Contains(filter, key []byte) bool 45 | } 46 | 47 | // FilterGenerator is the filter generator. 48 | type FilterGenerator interface { 49 | // Add adds a key to the filter generator. 50 | // 51 | // The key may become invalid after call to this method end, therefor 52 | // key must be copied if implementation require keeping key for later 53 | // use. The key should not modified directly, doing so may cause 54 | // undefined results. 55 | Add(key []byte) 56 | 57 | // Generate generates filters based on keys passed so far. After call 58 | // to Generate the filter generator maybe resetted, depends on implementation. 59 | Generate(b Buffer) 60 | } 61 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/util" 11 | ) 12 | 13 | // BasicArray is the interface that wraps basic Len and Search method. 14 | type BasicArray interface { 15 | // Len returns length of the array. 16 | Len() int 17 | 18 | // Search finds smallest index that point to a key that is greater 19 | // than or equal to the given key. 20 | Search(key []byte) int 21 | } 22 | 23 | // Array is the interface that wraps BasicArray and basic Index method. 24 | type Array interface { 25 | BasicArray 26 | 27 | // Index returns key/value pair with index of i. 28 | Index(i int) (key, value []byte) 29 | } 30 | 31 | // Array is the interface that wraps BasicArray and basic Get method. 32 | type ArrayIndexer interface { 33 | BasicArray 34 | 35 | // Get returns a new data iterator with index of i. 36 | Get(i int) Iterator 37 | } 38 | 39 | type basicArrayIterator struct { 40 | util.BasicReleaser 41 | array BasicArray 42 | pos int 43 | err error 44 | } 45 | 46 | func (i *basicArrayIterator) Valid() bool { 47 | return i.pos >= 0 && i.pos < i.array.Len() && !i.Released() 48 | } 49 | 50 | func (i *basicArrayIterator) First() bool { 51 | if i.Released() { 52 | i.err = ErrIterReleased 53 | return false 54 | } 55 | 56 | if i.array.Len() == 0 { 57 | i.pos = -1 58 | return false 59 | } 60 | i.pos = 0 61 | return true 62 | } 63 | 64 | func (i *basicArrayIterator) Last() bool { 65 | if i.Released() { 66 | i.err = ErrIterReleased 67 | return false 68 | } 69 | 70 | n := i.array.Len() 71 | if n == 0 { 72 | i.pos = 0 73 | return false 74 | } 75 | i.pos = n - 1 76 | return true 77 | } 78 | 79 | func (i *basicArrayIterator) Seek(key []byte) bool { 80 | if i.Released() { 81 | i.err = ErrIterReleased 82 | return false 83 | } 84 | 85 | n := i.array.Len() 86 | if n == 0 { 87 | i.pos = 0 88 | return false 89 | } 90 | i.pos = i.array.Search(key) 91 | if i.pos >= n { 92 | return false 93 | } 94 | return true 95 | } 96 | 97 | func (i *basicArrayIterator) Next() bool { 98 | if i.Released() { 99 | i.err = ErrIterReleased 100 | return false 101 | } 102 | 103 | i.pos++ 104 | if n := i.array.Len(); i.pos >= n { 105 | i.pos = n 106 | return false 107 | } 108 | return true 109 | } 110 | 111 | func (i *basicArrayIterator) Prev() bool { 112 | if i.Released() { 113 | i.err = ErrIterReleased 114 | return false 115 | } 116 | 117 | i.pos-- 118 | if i.pos < 0 { 119 | i.pos = -1 120 | return false 121 | } 122 | return true 123 | } 124 | 125 | func (i *basicArrayIterator) Error() error { return i.err } 126 | 127 | type arrayIterator struct { 128 | basicArrayIterator 129 | array Array 130 | pos int 131 | key, value []byte 132 | } 133 | 134 | func (i *arrayIterator) updateKV() { 135 | if i.pos == i.basicArrayIterator.pos { 136 | return 137 | } 138 | i.pos = i.basicArrayIterator.pos 139 | if i.Valid() { 140 | i.key, i.value = i.array.Index(i.pos) 141 | } else { 142 | i.key = nil 143 | i.value = nil 144 | } 145 | } 146 | 147 | func (i *arrayIterator) Key() []byte { 148 | i.updateKV() 149 | return i.key 150 | } 151 | 152 | func (i *arrayIterator) Value() []byte { 153 | i.updateKV() 154 | return i.value 155 | } 156 | 157 | type arrayIteratorIndexer struct { 158 | basicArrayIterator 159 | array ArrayIndexer 160 | } 161 | 162 | func (i *arrayIteratorIndexer) Get() Iterator { 163 | if i.Valid() { 164 | return i.array.Get(i.basicArrayIterator.pos) 165 | } 166 | return nil 167 | } 168 | 169 | // NewArrayIterator returns an iterator from the given array. 170 | func NewArrayIterator(array Array) Iterator { 171 | return &arrayIterator{ 172 | basicArrayIterator: basicArrayIterator{array: array, pos: -1}, 173 | array: array, 174 | pos: -1, 175 | } 176 | } 177 | 178 | // NewArrayIndexer returns an index iterator from the given array. 179 | func NewArrayIndexer(array ArrayIndexer) IteratorIndexer { 180 | return &arrayIteratorIndexer{ 181 | basicArrayIterator: basicArrayIterator{array: array, pos: -1}, 182 | array: array, 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package iterator 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/errors" 11 | "github.com/syndtr/goleveldb/leveldb/util" 12 | ) 13 | 14 | // IteratorIndexer is the interface that wraps CommonIterator and basic Get 15 | // method. IteratorIndexer provides index for indexed iterator. 16 | type IteratorIndexer interface { 17 | CommonIterator 18 | 19 | // Get returns a new data iterator for the current position, or nil if 20 | // done. 21 | Get() Iterator 22 | } 23 | 24 | type indexedIterator struct { 25 | util.BasicReleaser 26 | index IteratorIndexer 27 | strict bool 28 | 29 | data Iterator 30 | err error 31 | errf func(err error) 32 | closed bool 33 | } 34 | 35 | func (i *indexedIterator) setData() { 36 | if i.data != nil { 37 | i.data.Release() 38 | } 39 | i.data = i.index.Get() 40 | } 41 | 42 | func (i *indexedIterator) clearData() { 43 | if i.data != nil { 44 | i.data.Release() 45 | } 46 | i.data = nil 47 | } 48 | 49 | func (i *indexedIterator) indexErr() { 50 | if err := i.index.Error(); err != nil { 51 | if i.errf != nil { 52 | i.errf(err) 53 | } 54 | i.err = err 55 | } 56 | } 57 | 58 | func (i *indexedIterator) dataErr() bool { 59 | if err := i.data.Error(); err != nil { 60 | if i.errf != nil { 61 | i.errf(err) 62 | } 63 | if i.strict || !errors.IsCorrupted(err) { 64 | i.err = err 65 | return true 66 | } 67 | } 68 | return false 69 | } 70 | 71 | func (i *indexedIterator) Valid() bool { 72 | return i.data != nil && i.data.Valid() 73 | } 74 | 75 | func (i *indexedIterator) First() bool { 76 | if i.err != nil { 77 | return false 78 | } else if i.Released() { 79 | i.err = ErrIterReleased 80 | return false 81 | } 82 | 83 | if !i.index.First() { 84 | i.indexErr() 85 | i.clearData() 86 | return false 87 | } 88 | i.setData() 89 | return i.Next() 90 | } 91 | 92 | func (i *indexedIterator) Last() bool { 93 | if i.err != nil { 94 | return false 95 | } else if i.Released() { 96 | i.err = ErrIterReleased 97 | return false 98 | } 99 | 100 | if !i.index.Last() { 101 | i.indexErr() 102 | i.clearData() 103 | return false 104 | } 105 | i.setData() 106 | if !i.data.Last() { 107 | if i.dataErr() { 108 | return false 109 | } 110 | i.clearData() 111 | return i.Prev() 112 | } 113 | return true 114 | } 115 | 116 | func (i *indexedIterator) Seek(key []byte) bool { 117 | if i.err != nil { 118 | return false 119 | } else if i.Released() { 120 | i.err = ErrIterReleased 121 | return false 122 | } 123 | 124 | if !i.index.Seek(key) { 125 | i.indexErr() 126 | i.clearData() 127 | return false 128 | } 129 | i.setData() 130 | if !i.data.Seek(key) { 131 | if i.dataErr() { 132 | return false 133 | } 134 | i.clearData() 135 | return i.Next() 136 | } 137 | return true 138 | } 139 | 140 | func (i *indexedIterator) Next() bool { 141 | if i.err != nil { 142 | return false 143 | } else if i.Released() { 144 | i.err = ErrIterReleased 145 | return false 146 | } 147 | 148 | switch { 149 | case i.data != nil && !i.data.Next(): 150 | if i.dataErr() { 151 | return false 152 | } 153 | i.clearData() 154 | fallthrough 155 | case i.data == nil: 156 | if !i.index.Next() { 157 | i.indexErr() 158 | return false 159 | } 160 | i.setData() 161 | return i.Next() 162 | } 163 | return true 164 | } 165 | 166 | func (i *indexedIterator) Prev() bool { 167 | if i.err != nil { 168 | return false 169 | } else if i.Released() { 170 | i.err = ErrIterReleased 171 | return false 172 | } 173 | 174 | switch { 175 | case i.data != nil && !i.data.Prev(): 176 | if i.dataErr() { 177 | return false 178 | } 179 | i.clearData() 180 | fallthrough 181 | case i.data == nil: 182 | if !i.index.Prev() { 183 | i.indexErr() 184 | return false 185 | } 186 | i.setData() 187 | if !i.data.Last() { 188 | if i.dataErr() { 189 | return false 190 | } 191 | i.clearData() 192 | return i.Prev() 193 | } 194 | } 195 | return true 196 | } 197 | 198 | func (i *indexedIterator) Key() []byte { 199 | if i.data == nil { 200 | return nil 201 | } 202 | return i.data.Key() 203 | } 204 | 205 | func (i *indexedIterator) Value() []byte { 206 | if i.data == nil { 207 | return nil 208 | } 209 | return i.data.Value() 210 | } 211 | 212 | func (i *indexedIterator) Release() { 213 | i.clearData() 214 | i.index.Release() 215 | i.BasicReleaser.Release() 216 | } 217 | 218 | func (i *indexedIterator) Error() error { 219 | if i.err != nil { 220 | return i.err 221 | } 222 | if err := i.index.Error(); err != nil { 223 | return err 224 | } 225 | return nil 226 | } 227 | 228 | func (i *indexedIterator) SetErrorCallback(f func(err error)) { 229 | i.errf = f 230 | } 231 | 232 | // NewIndexedIterator returns an 'indexed iterator'. An index is iterator 233 | // that returns another iterator, a 'data iterator'. A 'data iterator' is the 234 | // iterator that contains actual key/value pairs. 235 | // 236 | // If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true) 237 | // won't be ignored and will halt 'indexed iterator', otherwise the iterator will 238 | // continue to the next 'data iterator'. Corruption on 'index iterator' will not be 239 | // ignored and will halt the iterator. 240 | func NewIndexedIterator(index IteratorIndexer, strict bool) Iterator { 241 | return &indexedIterator{index: index, strict: strict} 242 | } 243 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package iterator provides interface and implementation to traverse over 8 | // contents of a database. 9 | package iterator 10 | 11 | import ( 12 | "errors" 13 | 14 | "github.com/syndtr/goleveldb/leveldb/util" 15 | ) 16 | 17 | var ( 18 | ErrIterReleased = errors.New("leveldb/iterator: iterator released") 19 | ) 20 | 21 | // IteratorSeeker is the interface that wraps the 'seeks method'. 22 | type IteratorSeeker interface { 23 | // First moves the iterator to the first key/value pair. If the iterator 24 | // only contains one key/value pair then First and Last would moves 25 | // to the same key/value pair. 26 | // It returns whether such pair exist. 27 | First() bool 28 | 29 | // Last moves the iterator to the last key/value pair. If the iterator 30 | // only contains one key/value pair then First and Last would moves 31 | // to the same key/value pair. 32 | // It returns whether such pair exist. 33 | Last() bool 34 | 35 | // Seek moves the iterator to the first key/value pair whose key is greater 36 | // than or equal to the given key. 37 | // It returns whether such pair exist. 38 | // 39 | // It is safe to modify the contents of the argument after Seek returns. 40 | Seek(key []byte) bool 41 | 42 | // Next moves the iterator to the next key/value pair. 43 | // It returns false if the iterator is exhausted. 44 | Next() bool 45 | 46 | // Prev moves the iterator to the previous key/value pair. 47 | // It returns false if the iterator is exhausted. 48 | Prev() bool 49 | } 50 | 51 | // CommonIterator is the interface that wraps common iterator methods. 52 | type CommonIterator interface { 53 | IteratorSeeker 54 | 55 | // util.Releaser is the interface that wraps basic Release method. 56 | // When called Release will releases any resources associated with the 57 | // iterator. 58 | util.Releaser 59 | 60 | // util.ReleaseSetter is the interface that wraps the basic SetReleaser 61 | // method. 62 | util.ReleaseSetter 63 | 64 | // TODO: Remove this when ready. 65 | Valid() bool 66 | 67 | // Error returns any accumulated error. Exhausting all the key/value pairs 68 | // is not considered to be an error. 69 | Error() error 70 | } 71 | 72 | // Iterator iterates over a DB's key/value pairs in key order. 73 | // 74 | // When encounter an error any 'seeks method' will return false and will 75 | // yield no key/value pairs. The error can be queried by calling the Error 76 | // method. Calling Release is still necessary. 77 | // 78 | // An iterator must be released after use, but it is not necessary to read 79 | // an iterator until exhaustion. 80 | // Also, an iterator is not necessarily safe for concurrent use, but it is 81 | // safe to use multiple iterators concurrently, with each in a dedicated 82 | // goroutine. 83 | type Iterator interface { 84 | CommonIterator 85 | 86 | // Key returns the key of the current key/value pair, or nil if done. 87 | // The caller should not modify the contents of the returned slice, and 88 | // its contents may change on the next call to any 'seeks method'. 89 | Key() []byte 90 | 91 | // Value returns the value of the current key/value pair, or nil if done. 92 | // The caller should not modify the contents of the returned slice, and 93 | // its contents may change on the next call to any 'seeks method'. 94 | Value() []byte 95 | } 96 | 97 | // ErrorCallbackSetter is the interface that wraps basic SetErrorCallback 98 | // method. 99 | // 100 | // ErrorCallbackSetter implemented by indexed and merged iterator. 101 | type ErrorCallbackSetter interface { 102 | // SetErrorCallback allows set an error callback of the corresponding 103 | // iterator. Use nil to clear the callback. 104 | SetErrorCallback(f func(err error)) 105 | } 106 | 107 | type emptyIterator struct { 108 | util.BasicReleaser 109 | err error 110 | } 111 | 112 | func (i *emptyIterator) rErr() { 113 | if i.err == nil && i.Released() { 114 | i.err = ErrIterReleased 115 | } 116 | } 117 | 118 | func (*emptyIterator) Valid() bool { return false } 119 | func (i *emptyIterator) First() bool { i.rErr(); return false } 120 | func (i *emptyIterator) Last() bool { i.rErr(); return false } 121 | func (i *emptyIterator) Seek(key []byte) bool { i.rErr(); return false } 122 | func (i *emptyIterator) Next() bool { i.rErr(); return false } 123 | func (i *emptyIterator) Prev() bool { i.rErr(); return false } 124 | func (*emptyIterator) Key() []byte { return nil } 125 | func (*emptyIterator) Value() []byte { return nil } 126 | func (i *emptyIterator) Error() error { return i.err } 127 | 128 | // NewEmptyIterator creates an empty iterator. The err parameter can be 129 | // nil, but if not nil the given err will be returned by Error method. 130 | func NewEmptyIterator(err error) Iterator { 131 | return &emptyIterator{err: err} 132 | } 133 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/key.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "encoding/binary" 11 | "fmt" 12 | 13 | "github.com/syndtr/goleveldb/leveldb/errors" 14 | "github.com/syndtr/goleveldb/leveldb/storage" 15 | ) 16 | 17 | // ErrInternalKeyCorrupted records internal key corruption. 18 | type ErrInternalKeyCorrupted struct { 19 | Ikey []byte 20 | Reason string 21 | } 22 | 23 | func (e *ErrInternalKeyCorrupted) Error() string { 24 | return fmt.Sprintf("leveldb: internal key %q corrupted: %s", e.Ikey, e.Reason) 25 | } 26 | 27 | func newErrInternalKeyCorrupted(ikey []byte, reason string) error { 28 | return errors.NewErrCorrupted(storage.FileDesc{}, &ErrInternalKeyCorrupted{append([]byte{}, ikey...), reason}) 29 | } 30 | 31 | type keyType uint 32 | 33 | func (kt keyType) String() string { 34 | switch kt { 35 | case keyTypeDel: 36 | return "d" 37 | case keyTypeVal: 38 | return "v" 39 | } 40 | return fmt.Sprintf("", uint(kt)) 41 | } 42 | 43 | // Value types encoded as the last component of internal keys. 44 | // Don't modify; this value are saved to disk. 45 | const ( 46 | keyTypeDel = keyType(0) 47 | keyTypeVal = keyType(1) 48 | ) 49 | 50 | // keyTypeSeek defines the keyType that should be passed when constructing an 51 | // internal key for seeking to a particular sequence number (since we 52 | // sort sequence numbers in decreasing order and the value type is 53 | // embedded as the low 8 bits in the sequence number in internal keys, 54 | // we need to use the highest-numbered ValueType, not the lowest). 55 | const keyTypeSeek = keyTypeVal 56 | 57 | const ( 58 | // Maximum value possible for sequence number; the 8-bits are 59 | // used by value type, so its can packed together in single 60 | // 64-bit integer. 61 | keyMaxSeq = (uint64(1) << 56) - 1 62 | // Maximum value possible for packed sequence number and type. 63 | keyMaxNum = (keyMaxSeq << 8) | uint64(keyTypeSeek) 64 | ) 65 | 66 | // Maximum number encoded in bytes. 67 | var keyMaxNumBytes = make([]byte, 8) 68 | 69 | func init() { 70 | binary.LittleEndian.PutUint64(keyMaxNumBytes, keyMaxNum) 71 | } 72 | 73 | type internalKey []byte 74 | 75 | func makeInternalKey(dst, ukey []byte, seq uint64, kt keyType) internalKey { 76 | if seq > keyMaxSeq { 77 | panic("leveldb: invalid sequence number") 78 | } else if kt > keyTypeVal { 79 | panic("leveldb: invalid type") 80 | } 81 | 82 | dst = ensureBuffer(dst, len(ukey)+8) 83 | copy(dst, ukey) 84 | binary.LittleEndian.PutUint64(dst[len(ukey):], (seq<<8)|uint64(kt)) 85 | return internalKey(dst) 86 | } 87 | 88 | func parseInternalKey(ik []byte) (ukey []byte, seq uint64, kt keyType, err error) { 89 | if len(ik) < 8 { 90 | return nil, 0, 0, newErrInternalKeyCorrupted(ik, "invalid length") 91 | } 92 | num := binary.LittleEndian.Uint64(ik[len(ik)-8:]) 93 | seq, kt = uint64(num>>8), keyType(num&0xff) 94 | if kt > keyTypeVal { 95 | return nil, 0, 0, newErrInternalKeyCorrupted(ik, "invalid type") 96 | } 97 | ukey = ik[:len(ik)-8] 98 | return 99 | } 100 | 101 | func validInternalKey(ik []byte) bool { 102 | _, _, _, err := parseInternalKey(ik) 103 | return err == nil 104 | } 105 | 106 | func (ik internalKey) assert() { 107 | if ik == nil { 108 | panic("leveldb: nil internalKey") 109 | } 110 | if len(ik) < 8 { 111 | panic(fmt.Sprintf("leveldb: internal key %q, len=%d: invalid length", []byte(ik), len(ik))) 112 | } 113 | } 114 | 115 | func (ik internalKey) ukey() []byte { 116 | ik.assert() 117 | return ik[:len(ik)-8] 118 | } 119 | 120 | func (ik internalKey) num() uint64 { 121 | ik.assert() 122 | return binary.LittleEndian.Uint64(ik[len(ik)-8:]) 123 | } 124 | 125 | func (ik internalKey) parseNum() (seq uint64, kt keyType) { 126 | num := ik.num() 127 | seq, kt = uint64(num>>8), keyType(num&0xff) 128 | if kt > keyTypeVal { 129 | panic(fmt.Sprintf("leveldb: internal key %q, len=%d: invalid type %#x", []byte(ik), len(ik), kt)) 130 | } 131 | return 132 | } 133 | 134 | func (ik internalKey) String() string { 135 | if ik == nil { 136 | return "" 137 | } 138 | 139 | if ukey, seq, kt, err := parseInternalKey(ik); err == nil { 140 | return fmt.Sprintf("%s,%s%d", shorten(string(ukey)), kt, seq) 141 | } 142 | return fmt.Sprintf("", []byte(ik)) 143 | } 144 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/options.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "github.com/syndtr/goleveldb/leveldb/filter" 11 | "github.com/syndtr/goleveldb/leveldb/opt" 12 | ) 13 | 14 | func dupOptions(o *opt.Options) *opt.Options { 15 | newo := &opt.Options{} 16 | if o != nil { 17 | *newo = *o 18 | } 19 | if newo.Strict == 0 { 20 | newo.Strict = opt.DefaultStrict 21 | } 22 | return newo 23 | } 24 | 25 | func (s *session) setOptions(o *opt.Options) { 26 | no := dupOptions(o) 27 | // Alternative filters. 28 | if filters := o.GetAltFilters(); len(filters) > 0 { 29 | no.AltFilters = make([]filter.Filter, len(filters)) 30 | for i, filter := range filters { 31 | no.AltFilters[i] = &iFilter{filter} 32 | } 33 | } 34 | // Comparer. 35 | s.icmp = &iComparer{o.GetComparer()} 36 | no.Comparer = s.icmp 37 | // Filter. 38 | if filter := o.GetFilter(); filter != nil { 39 | no.Filter = &iFilter{filter} 40 | } 41 | 42 | s.o = &cachedOptions{Options: no} 43 | s.o.cache() 44 | } 45 | 46 | const optCachedLevel = 7 47 | 48 | type cachedOptions struct { 49 | *opt.Options 50 | 51 | compactionExpandLimit []int 52 | compactionGPOverlaps []int 53 | compactionSourceLimit []int 54 | compactionTableSize []int 55 | compactionTotalSize []int64 56 | } 57 | 58 | func (co *cachedOptions) cache() { 59 | co.compactionExpandLimit = make([]int, optCachedLevel) 60 | co.compactionGPOverlaps = make([]int, optCachedLevel) 61 | co.compactionSourceLimit = make([]int, optCachedLevel) 62 | co.compactionTableSize = make([]int, optCachedLevel) 63 | co.compactionTotalSize = make([]int64, optCachedLevel) 64 | 65 | for level := 0; level < optCachedLevel; level++ { 66 | co.compactionExpandLimit[level] = co.Options.GetCompactionExpandLimit(level) 67 | co.compactionGPOverlaps[level] = co.Options.GetCompactionGPOverlaps(level) 68 | co.compactionSourceLimit[level] = co.Options.GetCompactionSourceLimit(level) 69 | co.compactionTableSize[level] = co.Options.GetCompactionTableSize(level) 70 | co.compactionTotalSize[level] = co.Options.GetCompactionTotalSize(level) 71 | } 72 | } 73 | 74 | func (co *cachedOptions) GetCompactionExpandLimit(level int) int { 75 | if level < optCachedLevel { 76 | return co.compactionExpandLimit[level] 77 | } 78 | return co.Options.GetCompactionExpandLimit(level) 79 | } 80 | 81 | func (co *cachedOptions) GetCompactionGPOverlaps(level int) int { 82 | if level < optCachedLevel { 83 | return co.compactionGPOverlaps[level] 84 | } 85 | return co.Options.GetCompactionGPOverlaps(level) 86 | } 87 | 88 | func (co *cachedOptions) GetCompactionSourceLimit(level int) int { 89 | if level < optCachedLevel { 90 | return co.compactionSourceLimit[level] 91 | } 92 | return co.Options.GetCompactionSourceLimit(level) 93 | } 94 | 95 | func (co *cachedOptions) GetCompactionTableSize(level int) int { 96 | if level < optCachedLevel { 97 | return co.compactionTableSize[level] 98 | } 99 | return co.Options.GetCompactionTableSize(level) 100 | } 101 | 102 | func (co *cachedOptions) GetCompactionTotalSize(level int) int64 { 103 | if level < optCachedLevel { 104 | return co.compactionTotalSize[level] 105 | } 106 | return co.Options.GetCompactionTotalSize(level) 107 | } 108 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/session.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "fmt" 11 | "io" 12 | "os" 13 | "sync" 14 | 15 | "github.com/syndtr/goleveldb/leveldb/errors" 16 | "github.com/syndtr/goleveldb/leveldb/journal" 17 | "github.com/syndtr/goleveldb/leveldb/opt" 18 | "github.com/syndtr/goleveldb/leveldb/storage" 19 | ) 20 | 21 | // ErrManifestCorrupted records manifest corruption. This error will be 22 | // wrapped with errors.ErrCorrupted. 23 | type ErrManifestCorrupted struct { 24 | Field string 25 | Reason string 26 | } 27 | 28 | func (e *ErrManifestCorrupted) Error() string { 29 | return fmt.Sprintf("leveldb: manifest corrupted (field '%s'): %s", e.Field, e.Reason) 30 | } 31 | 32 | func newErrManifestCorrupted(fd storage.FileDesc, field, reason string) error { 33 | return errors.NewErrCorrupted(fd, &ErrManifestCorrupted{field, reason}) 34 | } 35 | 36 | // session represent a persistent database session. 37 | type session struct { 38 | // Need 64-bit alignment. 39 | stNextFileNum int64 // current unused file number 40 | stJournalNum int64 // current journal file number; need external synchronization 41 | stPrevJournalNum int64 // prev journal file number; no longer used; for compatibility with older version of leveldb 42 | stTempFileNum int64 43 | stSeqNum uint64 // last mem compacted seq; need external synchronization 44 | 45 | stor *iStorage 46 | storLock storage.Locker 47 | o *cachedOptions 48 | icmp *iComparer 49 | tops *tOps 50 | fileRef map[int64]int 51 | 52 | manifest *journal.Writer 53 | manifestWriter storage.Writer 54 | manifestFd storage.FileDesc 55 | 56 | stCompPtrs []internalKey // compaction pointers; need external synchronization 57 | stVersion *version // current version 58 | vmu sync.Mutex 59 | } 60 | 61 | // Creates new initialized session instance. 62 | func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) { 63 | if stor == nil { 64 | return nil, os.ErrInvalid 65 | } 66 | storLock, err := stor.Lock() 67 | if err != nil { 68 | return 69 | } 70 | s = &session{ 71 | stor: newIStorage(stor), 72 | storLock: storLock, 73 | fileRef: make(map[int64]int), 74 | } 75 | s.setOptions(o) 76 | s.tops = newTableOps(s) 77 | s.setVersion(newVersion(s)) 78 | s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed") 79 | return 80 | } 81 | 82 | // Close session. 83 | func (s *session) close() { 84 | s.tops.close() 85 | if s.manifest != nil { 86 | s.manifest.Close() 87 | } 88 | if s.manifestWriter != nil { 89 | s.manifestWriter.Close() 90 | } 91 | s.manifest = nil 92 | s.manifestWriter = nil 93 | s.setVersion(&version{s: s, closing: true}) 94 | } 95 | 96 | // Release session lock. 97 | func (s *session) release() { 98 | s.storLock.Unlock() 99 | } 100 | 101 | // Create a new database session; need external synchronization. 102 | func (s *session) create() error { 103 | // create manifest 104 | return s.newManifest(nil, nil) 105 | } 106 | 107 | // Recover a database session; need external synchronization. 108 | func (s *session) recover() (err error) { 109 | defer func() { 110 | if os.IsNotExist(err) { 111 | // Don't return os.ErrNotExist if the underlying storage contains 112 | // other files that belong to LevelDB. So the DB won't get trashed. 113 | if fds, _ := s.stor.List(storage.TypeAll); len(fds) > 0 { 114 | err = &errors.ErrCorrupted{Fd: storage.FileDesc{Type: storage.TypeManifest}, Err: &errors.ErrMissingFiles{}} 115 | } 116 | } 117 | }() 118 | 119 | fd, err := s.stor.GetMeta() 120 | if err != nil { 121 | return 122 | } 123 | 124 | reader, err := s.stor.Open(fd) 125 | if err != nil { 126 | return 127 | } 128 | defer reader.Close() 129 | 130 | var ( 131 | // Options. 132 | strict = s.o.GetStrict(opt.StrictManifest) 133 | 134 | jr = journal.NewReader(reader, dropper{s, fd}, strict, true) 135 | rec = &sessionRecord{} 136 | staging = s.stVersion.newStaging() 137 | ) 138 | for { 139 | var r io.Reader 140 | r, err = jr.Next() 141 | if err != nil { 142 | if err == io.EOF { 143 | err = nil 144 | break 145 | } 146 | return errors.SetFd(err, fd) 147 | } 148 | 149 | err = rec.decode(r) 150 | if err == nil { 151 | // save compact pointers 152 | for _, r := range rec.compPtrs { 153 | s.setCompPtr(r.level, internalKey(r.ikey)) 154 | } 155 | // commit record to version staging 156 | staging.commit(rec) 157 | } else { 158 | err = errors.SetFd(err, fd) 159 | if strict || !errors.IsCorrupted(err) { 160 | return 161 | } 162 | s.logf("manifest error: %v (skipped)", errors.SetFd(err, fd)) 163 | } 164 | rec.resetCompPtrs() 165 | rec.resetAddedTables() 166 | rec.resetDeletedTables() 167 | } 168 | 169 | switch { 170 | case !rec.has(recComparer): 171 | return newErrManifestCorrupted(fd, "comparer", "missing") 172 | case rec.comparer != s.icmp.uName(): 173 | return newErrManifestCorrupted(fd, "comparer", fmt.Sprintf("mismatch: want '%s', got '%s'", s.icmp.uName(), rec.comparer)) 174 | case !rec.has(recNextFileNum): 175 | return newErrManifestCorrupted(fd, "next-file-num", "missing") 176 | case !rec.has(recJournalNum): 177 | return newErrManifestCorrupted(fd, "journal-file-num", "missing") 178 | case !rec.has(recSeqNum): 179 | return newErrManifestCorrupted(fd, "seq-num", "missing") 180 | } 181 | 182 | s.manifestFd = fd 183 | s.setVersion(staging.finish()) 184 | s.setNextFileNum(rec.nextFileNum) 185 | s.recordCommited(rec) 186 | return nil 187 | } 188 | 189 | // Commit session; need external synchronization. 190 | func (s *session) commit(r *sessionRecord) (err error) { 191 | v := s.version() 192 | defer v.release() 193 | 194 | // spawn new version based on current version 195 | nv := v.spawn(r) 196 | 197 | if s.manifest == nil { 198 | // manifest journal writer not yet created, create one 199 | err = s.newManifest(r, nv) 200 | } else { 201 | err = s.flushManifest(r) 202 | } 203 | 204 | // finally, apply new version if no error rise 205 | if err == nil { 206 | s.setVersion(nv) 207 | } 208 | 209 | return 210 | } 211 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/storage.go: -------------------------------------------------------------------------------- 1 | package leveldb 2 | 3 | import ( 4 | "github.com/syndtr/goleveldb/leveldb/storage" 5 | "sync/atomic" 6 | ) 7 | 8 | type iStorage struct { 9 | storage.Storage 10 | read uint64 11 | write uint64 12 | } 13 | 14 | func (c *iStorage) Open(fd storage.FileDesc) (storage.Reader, error) { 15 | r, err := c.Storage.Open(fd) 16 | return &iStorageReader{r, c}, err 17 | } 18 | 19 | func (c *iStorage) Create(fd storage.FileDesc) (storage.Writer, error) { 20 | w, err := c.Storage.Create(fd) 21 | return &iStorageWriter{w, c}, err 22 | } 23 | 24 | func (c *iStorage) reads() uint64 { 25 | return atomic.LoadUint64(&c.read) 26 | } 27 | 28 | func (c *iStorage) writes() uint64 { 29 | return atomic.LoadUint64(&c.write) 30 | } 31 | 32 | // newIStorage returns the given storage wrapped by iStorage. 33 | func newIStorage(s storage.Storage) *iStorage { 34 | return &iStorage{s, 0, 0} 35 | } 36 | 37 | type iStorageReader struct { 38 | storage.Reader 39 | c *iStorage 40 | } 41 | 42 | func (r *iStorageReader) Read(p []byte) (n int, err error) { 43 | n, err = r.Reader.Read(p) 44 | atomic.AddUint64(&r.c.read, uint64(n)) 45 | return n, err 46 | } 47 | 48 | func (r *iStorageReader) ReadAt(p []byte, off int64) (n int, err error) { 49 | n, err = r.Reader.ReadAt(p, off) 50 | atomic.AddUint64(&r.c.read, uint64(n)) 51 | return n, err 52 | } 53 | 54 | type iStorageWriter struct { 55 | storage.Writer 56 | c *iStorage 57 | } 58 | 59 | func (w *iStorageWriter) Write(p []byte) (n int, err error) { 60 | n, err = w.Writer.Write(p) 61 | atomic.AddUint64(&w.c.write, uint64(n)) 62 | return n, err 63 | } 64 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_nacl.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // +build nacl 8 | 9 | package storage 10 | 11 | import ( 12 | "os" 13 | "syscall" 14 | ) 15 | 16 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 17 | return nil, syscall.ENOTSUP 18 | } 19 | 20 | func setFileLock(f *os.File, readOnly, lock bool) error { 21 | return syscall.ENOTSUP 22 | } 23 | 24 | func rename(oldpath, newpath string) error { 25 | return syscall.ENOTSUP 26 | } 27 | 28 | func isErrInvalid(err error) bool { 29 | return false 30 | } 31 | 32 | func syncDir(name string) error { 33 | return syscall.ENOTSUP 34 | } 35 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_plan9.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package storage 8 | 9 | import ( 10 | "os" 11 | ) 12 | 13 | type plan9FileLock struct { 14 | f *os.File 15 | } 16 | 17 | func (fl *plan9FileLock) release() error { 18 | return fl.f.Close() 19 | } 20 | 21 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 22 | var ( 23 | flag int 24 | perm os.FileMode 25 | ) 26 | if readOnly { 27 | flag = os.O_RDONLY 28 | } else { 29 | flag = os.O_RDWR 30 | perm = os.ModeExclusive 31 | } 32 | f, err := os.OpenFile(path, flag, perm) 33 | if os.IsNotExist(err) { 34 | f, err = os.OpenFile(path, flag|os.O_CREATE, perm|0644) 35 | } 36 | if err != nil { 37 | return 38 | } 39 | fl = &plan9FileLock{f: f} 40 | return 41 | } 42 | 43 | func rename(oldpath, newpath string) error { 44 | if _, err := os.Stat(newpath); err == nil { 45 | if err := os.Remove(newpath); err != nil { 46 | return err 47 | } 48 | } 49 | 50 | return os.Rename(oldpath, newpath) 51 | } 52 | 53 | func syncDir(name string) error { 54 | f, err := os.Open(name) 55 | if err != nil { 56 | return err 57 | } 58 | defer f.Close() 59 | if err := f.Sync(); err != nil { 60 | return err 61 | } 62 | return nil 63 | } 64 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_solaris.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // +build solaris 8 | 9 | package storage 10 | 11 | import ( 12 | "os" 13 | "syscall" 14 | ) 15 | 16 | type unixFileLock struct { 17 | f *os.File 18 | } 19 | 20 | func (fl *unixFileLock) release() error { 21 | if err := setFileLock(fl.f, false, false); err != nil { 22 | return err 23 | } 24 | return fl.f.Close() 25 | } 26 | 27 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 28 | var flag int 29 | if readOnly { 30 | flag = os.O_RDONLY 31 | } else { 32 | flag = os.O_RDWR 33 | } 34 | f, err := os.OpenFile(path, flag, 0) 35 | if os.IsNotExist(err) { 36 | f, err = os.OpenFile(path, flag|os.O_CREATE, 0644) 37 | } 38 | if err != nil { 39 | return 40 | } 41 | err = setFileLock(f, readOnly, true) 42 | if err != nil { 43 | f.Close() 44 | return 45 | } 46 | fl = &unixFileLock{f: f} 47 | return 48 | } 49 | 50 | func setFileLock(f *os.File, readOnly, lock bool) error { 51 | flock := syscall.Flock_t{ 52 | Type: syscall.F_UNLCK, 53 | Start: 0, 54 | Len: 0, 55 | Whence: 1, 56 | } 57 | if lock { 58 | if readOnly { 59 | flock.Type = syscall.F_RDLCK 60 | } else { 61 | flock.Type = syscall.F_WRLCK 62 | } 63 | } 64 | return syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &flock) 65 | } 66 | 67 | func rename(oldpath, newpath string) error { 68 | return os.Rename(oldpath, newpath) 69 | } 70 | 71 | func syncDir(name string) error { 72 | f, err := os.Open(name) 73 | if err != nil { 74 | return err 75 | } 76 | defer f.Close() 77 | if err := f.Sync(); err != nil { 78 | return err 79 | } 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // +build darwin dragonfly freebsd linux netbsd openbsd 8 | 9 | package storage 10 | 11 | import ( 12 | "os" 13 | "syscall" 14 | ) 15 | 16 | type unixFileLock struct { 17 | f *os.File 18 | } 19 | 20 | func (fl *unixFileLock) release() error { 21 | if err := setFileLock(fl.f, false, false); err != nil { 22 | return err 23 | } 24 | return fl.f.Close() 25 | } 26 | 27 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 28 | var flag int 29 | if readOnly { 30 | flag = os.O_RDONLY 31 | } else { 32 | flag = os.O_RDWR 33 | } 34 | f, err := os.OpenFile(path, flag, 0) 35 | if os.IsNotExist(err) { 36 | f, err = os.OpenFile(path, flag|os.O_CREATE, 0644) 37 | } 38 | if err != nil { 39 | return 40 | } 41 | err = setFileLock(f, readOnly, true) 42 | if err != nil { 43 | f.Close() 44 | return 45 | } 46 | fl = &unixFileLock{f: f} 47 | return 48 | } 49 | 50 | func setFileLock(f *os.File, readOnly, lock bool) error { 51 | how := syscall.LOCK_UN 52 | if lock { 53 | if readOnly { 54 | how = syscall.LOCK_SH 55 | } else { 56 | how = syscall.LOCK_EX 57 | } 58 | } 59 | return syscall.Flock(int(f.Fd()), how|syscall.LOCK_NB) 60 | } 61 | 62 | func rename(oldpath, newpath string) error { 63 | return os.Rename(oldpath, newpath) 64 | } 65 | 66 | func isErrInvalid(err error) bool { 67 | if err == os.ErrInvalid { 68 | return true 69 | } 70 | // Go < 1.8 71 | if syserr, ok := err.(*os.SyscallError); ok && syserr.Err == syscall.EINVAL { 72 | return true 73 | } 74 | // Go >= 1.8 returns *os.PathError instead 75 | if patherr, ok := err.(*os.PathError); ok && patherr.Err == syscall.EINVAL { 76 | return true 77 | } 78 | return false 79 | } 80 | 81 | func syncDir(name string) error { 82 | // As per fsync manpage, Linux seems to expect fsync on directory, however 83 | // some system don't support this, so we will ignore syscall.EINVAL. 84 | // 85 | // From fsync(2): 86 | // Calling fsync() does not necessarily ensure that the entry in the 87 | // directory containing the file has also reached disk. For that an 88 | // explicit fsync() on a file descriptor for the directory is also needed. 89 | f, err := os.Open(name) 90 | if err != nil { 91 | return err 92 | } 93 | defer f.Close() 94 | if err := f.Sync(); err != nil && !isErrInvalid(err) { 95 | return err 96 | } 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package storage 8 | 9 | import ( 10 | "syscall" 11 | "unsafe" 12 | ) 13 | 14 | var ( 15 | modkernel32 = syscall.NewLazyDLL("kernel32.dll") 16 | 17 | procMoveFileExW = modkernel32.NewProc("MoveFileExW") 18 | ) 19 | 20 | const ( 21 | _MOVEFILE_REPLACE_EXISTING = 1 22 | ) 23 | 24 | type windowsFileLock struct { 25 | fd syscall.Handle 26 | } 27 | 28 | func (fl *windowsFileLock) release() error { 29 | return syscall.Close(fl.fd) 30 | } 31 | 32 | func newFileLock(path string, readOnly bool) (fl fileLock, err error) { 33 | pathp, err := syscall.UTF16PtrFromString(path) 34 | if err != nil { 35 | return 36 | } 37 | var access, shareMode uint32 38 | if readOnly { 39 | access = syscall.GENERIC_READ 40 | shareMode = syscall.FILE_SHARE_READ 41 | } else { 42 | access = syscall.GENERIC_READ | syscall.GENERIC_WRITE 43 | } 44 | fd, err := syscall.CreateFile(pathp, access, shareMode, nil, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0) 45 | if err == syscall.ERROR_FILE_NOT_FOUND { 46 | fd, err = syscall.CreateFile(pathp, access, shareMode, nil, syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0) 47 | } 48 | if err != nil { 49 | return 50 | } 51 | fl = &windowsFileLock{fd: fd} 52 | return 53 | } 54 | 55 | func moveFileEx(from *uint16, to *uint16, flags uint32) error { 56 | r1, _, e1 := syscall.Syscall(procMoveFileExW.Addr(), 3, uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), uintptr(flags)) 57 | if r1 == 0 { 58 | if e1 != 0 { 59 | return error(e1) 60 | } 61 | return syscall.EINVAL 62 | } 63 | return nil 64 | } 65 | 66 | func rename(oldpath, newpath string) error { 67 | from, err := syscall.UTF16PtrFromString(oldpath) 68 | if err != nil { 69 | return err 70 | } 71 | to, err := syscall.UTF16PtrFromString(newpath) 72 | if err != nil { 73 | return err 74 | } 75 | return moveFileEx(from, to, _MOVEFILE_REPLACE_EXISTING) 76 | } 77 | 78 | func syncDir(name string) error { return nil } 79 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package storage 8 | 9 | import ( 10 | "bytes" 11 | "os" 12 | "sync" 13 | ) 14 | 15 | const typeShift = 4 16 | 17 | // Verify at compile-time that typeShift is large enough to cover all FileType 18 | // values by confirming that 0 == 0. 19 | var _ [0]struct{} = [TypeAll >> typeShift]struct{}{} 20 | 21 | type memStorageLock struct { 22 | ms *memStorage 23 | } 24 | 25 | func (lock *memStorageLock) Unlock() { 26 | ms := lock.ms 27 | ms.mu.Lock() 28 | defer ms.mu.Unlock() 29 | if ms.slock == lock { 30 | ms.slock = nil 31 | } 32 | return 33 | } 34 | 35 | // memStorage is a memory-backed storage. 36 | type memStorage struct { 37 | mu sync.Mutex 38 | slock *memStorageLock 39 | files map[uint64]*memFile 40 | meta FileDesc 41 | } 42 | 43 | // NewMemStorage returns a new memory-backed storage implementation. 44 | func NewMemStorage() Storage { 45 | return &memStorage{ 46 | files: make(map[uint64]*memFile), 47 | } 48 | } 49 | 50 | func (ms *memStorage) Lock() (Locker, error) { 51 | ms.mu.Lock() 52 | defer ms.mu.Unlock() 53 | if ms.slock != nil { 54 | return nil, ErrLocked 55 | } 56 | ms.slock = &memStorageLock{ms: ms} 57 | return ms.slock, nil 58 | } 59 | 60 | func (*memStorage) Log(str string) {} 61 | 62 | func (ms *memStorage) SetMeta(fd FileDesc) error { 63 | if !FileDescOk(fd) { 64 | return ErrInvalidFile 65 | } 66 | 67 | ms.mu.Lock() 68 | ms.meta = fd 69 | ms.mu.Unlock() 70 | return nil 71 | } 72 | 73 | func (ms *memStorage) GetMeta() (FileDesc, error) { 74 | ms.mu.Lock() 75 | defer ms.mu.Unlock() 76 | if ms.meta.Zero() { 77 | return FileDesc{}, os.ErrNotExist 78 | } 79 | return ms.meta, nil 80 | } 81 | 82 | func (ms *memStorage) List(ft FileType) ([]FileDesc, error) { 83 | ms.mu.Lock() 84 | var fds []FileDesc 85 | for x := range ms.files { 86 | fd := unpackFile(x) 87 | if fd.Type&ft != 0 { 88 | fds = append(fds, fd) 89 | } 90 | } 91 | ms.mu.Unlock() 92 | return fds, nil 93 | } 94 | 95 | func (ms *memStorage) Open(fd FileDesc) (Reader, error) { 96 | if !FileDescOk(fd) { 97 | return nil, ErrInvalidFile 98 | } 99 | 100 | ms.mu.Lock() 101 | defer ms.mu.Unlock() 102 | if m, exist := ms.files[packFile(fd)]; exist { 103 | if m.open { 104 | return nil, errFileOpen 105 | } 106 | m.open = true 107 | return &memReader{Reader: bytes.NewReader(m.Bytes()), ms: ms, m: m}, nil 108 | } 109 | return nil, os.ErrNotExist 110 | } 111 | 112 | func (ms *memStorage) Create(fd FileDesc) (Writer, error) { 113 | if !FileDescOk(fd) { 114 | return nil, ErrInvalidFile 115 | } 116 | 117 | x := packFile(fd) 118 | ms.mu.Lock() 119 | defer ms.mu.Unlock() 120 | m, exist := ms.files[x] 121 | if exist { 122 | if m.open { 123 | return nil, errFileOpen 124 | } 125 | m.Reset() 126 | } else { 127 | m = &memFile{} 128 | ms.files[x] = m 129 | } 130 | m.open = true 131 | return &memWriter{memFile: m, ms: ms}, nil 132 | } 133 | 134 | func (ms *memStorage) Remove(fd FileDesc) error { 135 | if !FileDescOk(fd) { 136 | return ErrInvalidFile 137 | } 138 | 139 | x := packFile(fd) 140 | ms.mu.Lock() 141 | defer ms.mu.Unlock() 142 | if _, exist := ms.files[x]; exist { 143 | delete(ms.files, x) 144 | return nil 145 | } 146 | return os.ErrNotExist 147 | } 148 | 149 | func (ms *memStorage) Rename(oldfd, newfd FileDesc) error { 150 | if !FileDescOk(oldfd) || !FileDescOk(newfd) { 151 | return ErrInvalidFile 152 | } 153 | if oldfd == newfd { 154 | return nil 155 | } 156 | 157 | oldx := packFile(oldfd) 158 | newx := packFile(newfd) 159 | ms.mu.Lock() 160 | defer ms.mu.Unlock() 161 | oldm, exist := ms.files[oldx] 162 | if !exist { 163 | return os.ErrNotExist 164 | } 165 | newm, exist := ms.files[newx] 166 | if (exist && newm.open) || oldm.open { 167 | return errFileOpen 168 | } 169 | delete(ms.files, oldx) 170 | ms.files[newx] = oldm 171 | return nil 172 | } 173 | 174 | func (*memStorage) Close() error { return nil } 175 | 176 | type memFile struct { 177 | bytes.Buffer 178 | open bool 179 | } 180 | 181 | type memReader struct { 182 | *bytes.Reader 183 | ms *memStorage 184 | m *memFile 185 | closed bool 186 | } 187 | 188 | func (mr *memReader) Close() error { 189 | mr.ms.mu.Lock() 190 | defer mr.ms.mu.Unlock() 191 | if mr.closed { 192 | return ErrClosed 193 | } 194 | mr.m.open = false 195 | return nil 196 | } 197 | 198 | type memWriter struct { 199 | *memFile 200 | ms *memStorage 201 | closed bool 202 | } 203 | 204 | func (*memWriter) Sync() error { return nil } 205 | 206 | func (mw *memWriter) Close() error { 207 | mw.ms.mu.Lock() 208 | defer mw.ms.mu.Unlock() 209 | if mw.closed { 210 | return ErrClosed 211 | } 212 | mw.memFile.open = false 213 | return nil 214 | } 215 | 216 | func packFile(fd FileDesc) uint64 { 217 | return uint64(fd.Num)<> typeShift)} 222 | } 223 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/storage/storage.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package storage provides storage abstraction for LevelDB. 8 | package storage 9 | 10 | import ( 11 | "errors" 12 | "fmt" 13 | "io" 14 | ) 15 | 16 | // FileType represent a file type. 17 | type FileType int 18 | 19 | // File types. 20 | const ( 21 | TypeManifest FileType = 1 << iota 22 | TypeJournal 23 | TypeTable 24 | TypeTemp 25 | 26 | TypeAll = TypeManifest | TypeJournal | TypeTable | TypeTemp 27 | ) 28 | 29 | func (t FileType) String() string { 30 | switch t { 31 | case TypeManifest: 32 | return "manifest" 33 | case TypeJournal: 34 | return "journal" 35 | case TypeTable: 36 | return "table" 37 | case TypeTemp: 38 | return "temp" 39 | } 40 | return fmt.Sprintf("", t) 41 | } 42 | 43 | // Common error. 44 | var ( 45 | ErrInvalidFile = errors.New("leveldb/storage: invalid file for argument") 46 | ErrLocked = errors.New("leveldb/storage: already locked") 47 | ErrClosed = errors.New("leveldb/storage: closed") 48 | ) 49 | 50 | // ErrCorrupted is the type that wraps errors that indicate corruption of 51 | // a file. Package storage has its own type instead of using 52 | // errors.ErrCorrupted to prevent circular import. 53 | type ErrCorrupted struct { 54 | Fd FileDesc 55 | Err error 56 | } 57 | 58 | func isCorrupted(err error) bool { 59 | switch err.(type) { 60 | case *ErrCorrupted: 61 | return true 62 | } 63 | return false 64 | } 65 | 66 | func (e *ErrCorrupted) Error() string { 67 | if !e.Fd.Zero() { 68 | return fmt.Sprintf("%v [file=%v]", e.Err, e.Fd) 69 | } 70 | return e.Err.Error() 71 | } 72 | 73 | // Syncer is the interface that wraps basic Sync method. 74 | type Syncer interface { 75 | // Sync commits the current contents of the file to stable storage. 76 | Sync() error 77 | } 78 | 79 | // Reader is the interface that groups the basic Read, Seek, ReadAt and Close 80 | // methods. 81 | type Reader interface { 82 | io.ReadSeeker 83 | io.ReaderAt 84 | io.Closer 85 | } 86 | 87 | // Writer is the interface that groups the basic Write, Sync and Close 88 | // methods. 89 | type Writer interface { 90 | io.WriteCloser 91 | Syncer 92 | } 93 | 94 | // Locker is the interface that wraps Unlock method. 95 | type Locker interface { 96 | Unlock() 97 | } 98 | 99 | // FileDesc is a 'file descriptor'. 100 | type FileDesc struct { 101 | Type FileType 102 | Num int64 103 | } 104 | 105 | func (fd FileDesc) String() string { 106 | switch fd.Type { 107 | case TypeManifest: 108 | return fmt.Sprintf("MANIFEST-%06d", fd.Num) 109 | case TypeJournal: 110 | return fmt.Sprintf("%06d.log", fd.Num) 111 | case TypeTable: 112 | return fmt.Sprintf("%06d.ldb", fd.Num) 113 | case TypeTemp: 114 | return fmt.Sprintf("%06d.tmp", fd.Num) 115 | default: 116 | return fmt.Sprintf("%#x-%d", fd.Type, fd.Num) 117 | } 118 | } 119 | 120 | // Zero returns true if fd == (FileDesc{}). 121 | func (fd FileDesc) Zero() bool { 122 | return fd == (FileDesc{}) 123 | } 124 | 125 | // FileDescOk returns true if fd is a valid 'file descriptor'. 126 | func FileDescOk(fd FileDesc) bool { 127 | switch fd.Type { 128 | case TypeManifest: 129 | case TypeJournal: 130 | case TypeTable: 131 | case TypeTemp: 132 | default: 133 | return false 134 | } 135 | return fd.Num >= 0 136 | } 137 | 138 | // Storage is the storage. A storage instance must be safe for concurrent use. 139 | type Storage interface { 140 | // Lock locks the storage. Any subsequent attempt to call Lock will fail 141 | // until the last lock released. 142 | // Caller should call Unlock method after use. 143 | Lock() (Locker, error) 144 | 145 | // Log logs a string. This is used for logging. 146 | // An implementation may write to a file, stdout or simply do nothing. 147 | Log(str string) 148 | 149 | // SetMeta store 'file descriptor' that can later be acquired using GetMeta 150 | // method. The 'file descriptor' should point to a valid file. 151 | // SetMeta should be implemented in such way that changes should happen 152 | // atomically. 153 | SetMeta(fd FileDesc) error 154 | 155 | // GetMeta returns 'file descriptor' stored in meta. The 'file descriptor' 156 | // can be updated using SetMeta method. 157 | // Returns os.ErrNotExist if meta doesn't store any 'file descriptor', or 158 | // 'file descriptor' point to nonexistent file. 159 | GetMeta() (FileDesc, error) 160 | 161 | // List returns file descriptors that match the given file types. 162 | // The file types may be OR'ed together. 163 | List(ft FileType) ([]FileDesc, error) 164 | 165 | // Open opens file with the given 'file descriptor' read-only. 166 | // Returns os.ErrNotExist error if the file does not exist. 167 | // Returns ErrClosed if the underlying storage is closed. 168 | Open(fd FileDesc) (Reader, error) 169 | 170 | // Create creates file with the given 'file descriptor', truncate if already 171 | // exist and opens write-only. 172 | // Returns ErrClosed if the underlying storage is closed. 173 | Create(fd FileDesc) (Writer, error) 174 | 175 | // Remove removes file with the given 'file descriptor'. 176 | // Returns ErrClosed if the underlying storage is closed. 177 | Remove(fd FileDesc) error 178 | 179 | // Rename renames file from oldfd to newfd. 180 | // Returns ErrClosed if the underlying storage is closed. 181 | Rename(oldfd, newfd FileDesc) error 182 | 183 | // Close closes the storage. 184 | // It is valid to call Close multiple times. Other methods should not be 185 | // called after the storage has been closed. 186 | Close() error 187 | } 188 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package leveldb 8 | 9 | import ( 10 | "fmt" 11 | "sort" 12 | 13 | "github.com/syndtr/goleveldb/leveldb/storage" 14 | ) 15 | 16 | func shorten(str string) string { 17 | if len(str) <= 8 { 18 | return str 19 | } 20 | return str[:3] + ".." + str[len(str)-3:] 21 | } 22 | 23 | var bunits = [...]string{"", "Ki", "Mi", "Gi", "Ti"} 24 | 25 | func shortenb(bytes int) string { 26 | i := 0 27 | for ; bytes > 1024 && i < 4; i++ { 28 | bytes /= 1024 29 | } 30 | return fmt.Sprintf("%d%sB", bytes, bunits[i]) 31 | } 32 | 33 | func sshortenb(bytes int) string { 34 | if bytes == 0 { 35 | return "~" 36 | } 37 | sign := "+" 38 | if bytes < 0 { 39 | sign = "-" 40 | bytes *= -1 41 | } 42 | i := 0 43 | for ; bytes > 1024 && i < 4; i++ { 44 | bytes /= 1024 45 | } 46 | return fmt.Sprintf("%s%d%sB", sign, bytes, bunits[i]) 47 | } 48 | 49 | func sint(x int) string { 50 | if x == 0 { 51 | return "~" 52 | } 53 | sign := "+" 54 | if x < 0 { 55 | sign = "-" 56 | x *= -1 57 | } 58 | return fmt.Sprintf("%s%d", sign, x) 59 | } 60 | 61 | func minInt(a, b int) int { 62 | if a < b { 63 | return a 64 | } 65 | return b 66 | } 67 | 68 | func maxInt(a, b int) int { 69 | if a > b { 70 | return a 71 | } 72 | return b 73 | } 74 | 75 | type fdSorter []storage.FileDesc 76 | 77 | func (p fdSorter) Len() int { 78 | return len(p) 79 | } 80 | 81 | func (p fdSorter) Less(i, j int) bool { 82 | return p[i].Num < p[j].Num 83 | } 84 | 85 | func (p fdSorter) Swap(i, j int) { 86 | p[i], p[j] = p[j], p[i] 87 | } 88 | 89 | func sortFds(fds []storage.FileDesc) { 90 | sort.Sort(fdSorter(fds)) 91 | } 92 | 93 | func ensureBuffer(b []byte, n int) []byte { 94 | if cap(b) < n { 95 | return make([]byte, n) 96 | } 97 | return b[:n] 98 | } 99 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package util 8 | 9 | import ( 10 | "fmt" 11 | "sync" 12 | "sync/atomic" 13 | "time" 14 | ) 15 | 16 | type buffer struct { 17 | b []byte 18 | miss int 19 | } 20 | 21 | // BufferPool is a 'buffer pool'. 22 | type BufferPool struct { 23 | pool [6]chan []byte 24 | size [5]uint32 25 | sizeMiss [5]uint32 26 | sizeHalf [5]uint32 27 | baseline [4]int 28 | baseline0 int 29 | 30 | mu sync.RWMutex 31 | closed bool 32 | closeC chan struct{} 33 | 34 | get uint32 35 | put uint32 36 | half uint32 37 | less uint32 38 | equal uint32 39 | greater uint32 40 | miss uint32 41 | } 42 | 43 | func (p *BufferPool) poolNum(n int) int { 44 | if n <= p.baseline0 && n > p.baseline0/2 { 45 | return 0 46 | } 47 | for i, x := range p.baseline { 48 | if n <= x { 49 | return i + 1 50 | } 51 | } 52 | return len(p.baseline) + 1 53 | } 54 | 55 | // Get returns buffer with length of n. 56 | func (p *BufferPool) Get(n int) []byte { 57 | if p == nil { 58 | return make([]byte, n) 59 | } 60 | 61 | p.mu.RLock() 62 | defer p.mu.RUnlock() 63 | 64 | if p.closed { 65 | return make([]byte, n) 66 | } 67 | 68 | atomic.AddUint32(&p.get, 1) 69 | 70 | poolNum := p.poolNum(n) 71 | pool := p.pool[poolNum] 72 | if poolNum == 0 { 73 | // Fast path. 74 | select { 75 | case b := <-pool: 76 | switch { 77 | case cap(b) > n: 78 | if cap(b)-n >= n { 79 | atomic.AddUint32(&p.half, 1) 80 | select { 81 | case pool <- b: 82 | default: 83 | } 84 | return make([]byte, n) 85 | } else { 86 | atomic.AddUint32(&p.less, 1) 87 | return b[:n] 88 | } 89 | case cap(b) == n: 90 | atomic.AddUint32(&p.equal, 1) 91 | return b[:n] 92 | default: 93 | atomic.AddUint32(&p.greater, 1) 94 | } 95 | default: 96 | atomic.AddUint32(&p.miss, 1) 97 | } 98 | 99 | return make([]byte, n, p.baseline0) 100 | } else { 101 | sizePtr := &p.size[poolNum-1] 102 | 103 | select { 104 | case b := <-pool: 105 | switch { 106 | case cap(b) > n: 107 | if cap(b)-n >= n { 108 | atomic.AddUint32(&p.half, 1) 109 | sizeHalfPtr := &p.sizeHalf[poolNum-1] 110 | if atomic.AddUint32(sizeHalfPtr, 1) == 20 { 111 | atomic.StoreUint32(sizePtr, uint32(cap(b)/2)) 112 | atomic.StoreUint32(sizeHalfPtr, 0) 113 | } else { 114 | select { 115 | case pool <- b: 116 | default: 117 | } 118 | } 119 | return make([]byte, n) 120 | } else { 121 | atomic.AddUint32(&p.less, 1) 122 | return b[:n] 123 | } 124 | case cap(b) == n: 125 | atomic.AddUint32(&p.equal, 1) 126 | return b[:n] 127 | default: 128 | atomic.AddUint32(&p.greater, 1) 129 | if uint32(cap(b)) >= atomic.LoadUint32(sizePtr) { 130 | select { 131 | case pool <- b: 132 | default: 133 | } 134 | } 135 | } 136 | default: 137 | atomic.AddUint32(&p.miss, 1) 138 | } 139 | 140 | if size := atomic.LoadUint32(sizePtr); uint32(n) > size { 141 | if size == 0 { 142 | atomic.CompareAndSwapUint32(sizePtr, 0, uint32(n)) 143 | } else { 144 | sizeMissPtr := &p.sizeMiss[poolNum-1] 145 | if atomic.AddUint32(sizeMissPtr, 1) == 20 { 146 | atomic.StoreUint32(sizePtr, uint32(n)) 147 | atomic.StoreUint32(sizeMissPtr, 0) 148 | } 149 | } 150 | return make([]byte, n) 151 | } else { 152 | return make([]byte, n, size) 153 | } 154 | } 155 | } 156 | 157 | // Put adds given buffer to the pool. 158 | func (p *BufferPool) Put(b []byte) { 159 | if p == nil { 160 | return 161 | } 162 | 163 | p.mu.RLock() 164 | defer p.mu.RUnlock() 165 | 166 | if p.closed { 167 | return 168 | } 169 | 170 | atomic.AddUint32(&p.put, 1) 171 | 172 | pool := p.pool[p.poolNum(cap(b))] 173 | select { 174 | case pool <- b: 175 | default: 176 | } 177 | 178 | } 179 | 180 | func (p *BufferPool) Close() { 181 | if p == nil { 182 | return 183 | } 184 | 185 | p.mu.Lock() 186 | if !p.closed { 187 | p.closed = true 188 | p.closeC <- struct{}{} 189 | } 190 | p.mu.Unlock() 191 | } 192 | 193 | func (p *BufferPool) String() string { 194 | if p == nil { 195 | return "" 196 | } 197 | 198 | return fmt.Sprintf("BufferPool{B·%d Z·%v Zm·%v Zh·%v G·%d P·%d H·%d <·%d =·%d >·%d M·%d}", 199 | p.baseline0, p.size, p.sizeMiss, p.sizeHalf, p.get, p.put, p.half, p.less, p.equal, p.greater, p.miss) 200 | } 201 | 202 | func (p *BufferPool) drain() { 203 | ticker := time.NewTicker(2 * time.Second) 204 | defer ticker.Stop() 205 | for { 206 | select { 207 | case <-ticker.C: 208 | for _, ch := range p.pool { 209 | select { 210 | case <-ch: 211 | default: 212 | } 213 | } 214 | case <-p.closeC: 215 | close(p.closeC) 216 | for _, ch := range p.pool { 217 | close(ch) 218 | } 219 | return 220 | } 221 | } 222 | } 223 | 224 | // NewBufferPool creates a new initialized 'buffer pool'. 225 | func NewBufferPool(baseline int) *BufferPool { 226 | if baseline <= 0 { 227 | panic("baseline can't be <= 0") 228 | } 229 | p := &BufferPool{ 230 | baseline0: baseline, 231 | baseline: [...]int{baseline / 4, baseline / 2, baseline * 2, baseline * 4}, 232 | closeC: make(chan struct{}, 1), 233 | } 234 | for i, cap := range []int{2, 2, 4, 4, 2, 1} { 235 | p.pool[i] = make(chan []byte, cap) 236 | } 237 | go p.drain() 238 | return p 239 | } 240 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/util/crc32.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The LevelDB-Go Authors. All rights reserved. 2 | // 3 | // Use of this source code is governed by a BSD-style license that can be 4 | // found in the LICENSE file. 5 | 6 | package util 7 | 8 | import ( 9 | "hash/crc32" 10 | ) 11 | 12 | var table = crc32.MakeTable(crc32.Castagnoli) 13 | 14 | // CRC is a CRC-32 checksum computed using Castagnoli's polynomial. 15 | type CRC uint32 16 | 17 | // NewCRC creates a new crc based on the given bytes. 18 | func NewCRC(b []byte) CRC { 19 | return CRC(0).Update(b) 20 | } 21 | 22 | // Update updates the crc with the given bytes. 23 | func (c CRC) Update(b []byte) CRC { 24 | return CRC(crc32.Update(uint32(c), table, b)) 25 | } 26 | 27 | // Value returns a masked crc. 28 | func (c CRC) Value() uint32 { 29 | return uint32(c>>15|c<<17) + 0xa282ead8 30 | } 31 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/util/hash.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package util 8 | 9 | import ( 10 | "encoding/binary" 11 | ) 12 | 13 | // Hash return hash of the given data. 14 | func Hash(data []byte, seed uint32) uint32 { 15 | // Similar to murmur hash 16 | const ( 17 | m = uint32(0xc6a4a793) 18 | r = uint32(24) 19 | ) 20 | var ( 21 | h = seed ^ (uint32(len(data)) * m) 22 | i int 23 | ) 24 | 25 | for n := len(data) - len(data)%4; i < n; i += 4 { 26 | h += binary.LittleEndian.Uint32(data[i:]) 27 | h *= m 28 | h ^= (h >> 16) 29 | } 30 | 31 | switch len(data) - i { 32 | default: 33 | panic("not reached") 34 | case 3: 35 | h += uint32(data[i+2]) << 16 36 | fallthrough 37 | case 2: 38 | h += uint32(data[i+1]) << 8 39 | fallthrough 40 | case 1: 41 | h += uint32(data[i]) 42 | h *= m 43 | h ^= (h >> r) 44 | case 0: 45 | } 46 | 47 | return h 48 | } 49 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/util/range.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | package util 8 | 9 | // Range is a key range. 10 | type Range struct { 11 | // Start of the key range, include in the range. 12 | Start []byte 13 | 14 | // Limit of the key range, not include in the range. 15 | Limit []byte 16 | } 17 | 18 | // BytesPrefix returns key range that satisfy the given prefix. 19 | // This only applicable for the standard 'bytes comparer'. 20 | func BytesPrefix(prefix []byte) *Range { 21 | var limit []byte 22 | for i := len(prefix) - 1; i >= 0; i-- { 23 | c := prefix[i] 24 | if c < 0xff { 25 | limit = make([]byte, i+1) 26 | copy(limit, prefix) 27 | limit[i] = c + 1 28 | break 29 | } 30 | } 31 | return &Range{prefix, limit} 32 | } 33 | -------------------------------------------------------------------------------- /vendor/github.com/syndtr/goleveldb/leveldb/util/util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2013, Suryandaru Triandana 2 | // All rights reserved. 3 | // 4 | // Use of this source code is governed by a BSD-style license that can be 5 | // found in the LICENSE file. 6 | 7 | // Package util provides utilities used throughout leveldb. 8 | package util 9 | 10 | import ( 11 | "errors" 12 | ) 13 | 14 | var ( 15 | ErrReleased = errors.New("leveldb: resource already relesed") 16 | ErrHasReleaser = errors.New("leveldb: releaser already defined") 17 | ) 18 | 19 | // Releaser is the interface that wraps the basic Release method. 20 | type Releaser interface { 21 | // Release releases associated resources. Release should always success 22 | // and can be called multiple times without causing error. 23 | Release() 24 | } 25 | 26 | // ReleaseSetter is the interface that wraps the basic SetReleaser method. 27 | type ReleaseSetter interface { 28 | // SetReleaser associates the given releaser to the resources. The 29 | // releaser will be called once coresponding resources released. 30 | // Calling SetReleaser with nil will clear the releaser. 31 | // 32 | // This will panic if a releaser already present or coresponding 33 | // resource is already released. Releaser should be cleared first 34 | // before assigned a new one. 35 | SetReleaser(releaser Releaser) 36 | } 37 | 38 | // BasicReleaser provides basic implementation of Releaser and ReleaseSetter. 39 | type BasicReleaser struct { 40 | releaser Releaser 41 | released bool 42 | } 43 | 44 | // Released returns whether Release method already called. 45 | func (r *BasicReleaser) Released() bool { 46 | return r.released 47 | } 48 | 49 | // Release implements Releaser.Release. 50 | func (r *BasicReleaser) Release() { 51 | if !r.released { 52 | if r.releaser != nil { 53 | r.releaser.Release() 54 | r.releaser = nil 55 | } 56 | r.released = true 57 | } 58 | } 59 | 60 | // SetReleaser implements ReleaseSetter.SetReleaser. 61 | func (r *BasicReleaser) SetReleaser(releaser Releaser) { 62 | if r.released { 63 | panic(ErrReleased) 64 | } 65 | if r.releaser != nil && releaser != nil { 66 | panic(ErrHasReleaser) 67 | } 68 | r.releaser = releaser 69 | } 70 | 71 | type NoopReleaser struct{} 72 | 73 | func (NoopReleaser) Release() {} 74 | -------------------------------------------------------------------------------- /vendor/github.com/tus/tusd/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013-2017 Transloadit Ltd and Contributors 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 7 | of the Software, and to permit persons to whom the Software is furnished to do 8 | so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /vendor/github.com/tus/tusd/internal/uid/uid.go: -------------------------------------------------------------------------------- 1 | package uid 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/hex" 6 | "io" 7 | ) 8 | 9 | // uid returns a unique id. These ids consist of 128 bits from a 10 | // cryptographically strong pseudo-random generator and are like uuids, but 11 | // without the dashes and significant bits. 12 | // 13 | // See: http://en.wikipedia.org/wiki/UUID#Random_UUID_probability_of_duplicates 14 | func Uid() string { 15 | id := make([]byte, 16) 16 | _, err := io.ReadFull(rand.Reader, id) 17 | if err != nil { 18 | // This is probably an appropriate way to handle errors from our source 19 | // for random bits. 20 | panic(err) 21 | } 22 | return hex.EncodeToString(id) 23 | } 24 | -------------------------------------------------------------------------------- /vendor/github.com/tus/tusd/pkg/handler/composer.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | // StoreComposer represents a composable data store. It consists of the core 4 | // data store and optional extensions. Please consult the package's overview 5 | // for a more detailed introduction in how to use this structure. 6 | type StoreComposer struct { 7 | Core DataStore 8 | 9 | UsesTerminater bool 10 | Terminater TerminaterDataStore 11 | UsesLocker bool 12 | Locker Locker 13 | UsesConcater bool 14 | Concater ConcaterDataStore 15 | UsesLengthDeferrer bool 16 | LengthDeferrer LengthDeferrerDataStore 17 | } 18 | 19 | // NewStoreComposer creates a new and empty store composer. 20 | func NewStoreComposer() *StoreComposer { 21 | return &StoreComposer{} 22 | } 23 | 24 | // Capabilities returns a string representing the provided extensions in a 25 | // human-readable format meant for debugging. 26 | func (store *StoreComposer) Capabilities() string { 27 | str := "Core: " 28 | 29 | if store.Core != nil { 30 | str += "✓" 31 | } else { 32 | str += "✗" 33 | } 34 | 35 | str += ` Terminater: ` 36 | if store.UsesTerminater { 37 | str += "✓" 38 | } else { 39 | str += "✗" 40 | } 41 | str += ` Locker: ` 42 | if store.UsesLocker { 43 | str += "✓" 44 | } else { 45 | str += "✗" 46 | } 47 | str += ` Concater: ` 48 | if store.UsesConcater { 49 | str += "✓" 50 | } else { 51 | str += "✗" 52 | } 53 | str += ` LengthDeferrer: ` 54 | if store.UsesLengthDeferrer { 55 | str += "✓" 56 | } else { 57 | str += "✗" 58 | } 59 | 60 | return str 61 | } 62 | 63 | // UseCore will set the used core data store. If the argument is nil, the 64 | // property will be unset. 65 | func (store *StoreComposer) UseCore(core DataStore) { 66 | store.Core = core 67 | } 68 | 69 | func (store *StoreComposer) UseTerminater(ext TerminaterDataStore) { 70 | store.UsesTerminater = ext != nil 71 | store.Terminater = ext 72 | } 73 | 74 | func (store *StoreComposer) UseLocker(ext Locker) { 75 | store.UsesLocker = ext != nil 76 | store.Locker = ext 77 | } 78 | 79 | func (store *StoreComposer) UseConcater(ext ConcaterDataStore) { 80 | store.UsesConcater = ext != nil 81 | store.Concater = ext 82 | } 83 | 84 | func (store *StoreComposer) UseLengthDeferrer(ext LengthDeferrerDataStore) { 85 | store.UsesLengthDeferrer = ext != nil 86 | store.LengthDeferrer = ext 87 | } 88 | -------------------------------------------------------------------------------- /vendor/github.com/tus/tusd/pkg/handler/composer.mgo: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | #define USE_FUNC(TYPE) \ 4 | func (store *StoreComposer) Use ## TYPE(ext TYPE ## DataStore) { \ 5 | store.Uses ## TYPE = ext != nil; \ 6 | store.TYPE = ext; \ 7 | } 8 | 9 | #define USE_FIELD(TYPE) Uses ## TYPE bool; \ 10 | TYPE TYPE ## DataStore 11 | 12 | #define USE_FROM(TYPE) if mod, ok := store.(TYPE ## DataStore); ok { \ 13 | composer.Use ## TYPE (mod) \ 14 | } 15 | 16 | #define USE_CAP(TYPE) str += ` TYPE: `; \ 17 | if store.Uses ## TYPE { \ 18 | str += "✓" \ 19 | } else { \ 20 | str += "✗" \ 21 | } 22 | 23 | // StoreComposer represents a composable data store. It consists of the core 24 | // data store and optional extensions. Please consult the package's overview 25 | // for a more detailed introduction in how to use this structure. 26 | type StoreComposer struct { 27 | Core DataStore 28 | 29 | USE_FIELD(Terminater) 30 | USE_FIELD(Finisher) 31 | USE_FIELD(Locker) 32 | USE_FIELD(GetReader) 33 | USE_FIELD(Concater) 34 | USE_FIELD(LengthDeferrer) 35 | } 36 | 37 | // NewStoreComposer creates a new and empty store composer. 38 | func NewStoreComposer() *StoreComposer { 39 | return &StoreComposer{} 40 | } 41 | 42 | // Capabilities returns a string representing the provided extensions in a 43 | // human-readable format meant for debugging. 44 | func (store *StoreComposer) Capabilities() string { 45 | str := "Core: " 46 | 47 | if store.Core != nil { 48 | str += "✓" 49 | } else { 50 | str += "✗" 51 | } 52 | 53 | USE_CAP(Terminater) 54 | USE_CAP(Finisher) 55 | USE_CAP(Locker) 56 | USE_CAP(GetReader) 57 | USE_CAP(Concater) 58 | USE_CAP(LengthDeferrer) 59 | 60 | return str 61 | } 62 | 63 | // UseCore will set the used core data store. If the argument is nil, the 64 | // property will be unset. 65 | func (store *StoreComposer) UseCore(core DataStore) { 66 | store.Core = core 67 | } 68 | 69 | USE_FUNC(Terminater) 70 | USE_FUNC(Finisher) 71 | USE_FUNC(Locker) 72 | USE_FUNC(GetReader) 73 | USE_FUNC(Concater) 74 | USE_FUNC(LengthDeferrer) 75 | -------------------------------------------------------------------------------- /vendor/github.com/tus/tusd/pkg/handler/config.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "errors" 5 | "log" 6 | "net/url" 7 | "os" 8 | ) 9 | 10 | // Config provides a way to configure the Handler depending on your needs. 11 | type Config struct { 12 | // StoreComposer points to the store composer from which the core data store 13 | // and optional dependencies should be taken. May only be nil if DataStore is 14 | // set. 15 | // TODO: Remove pointer? 16 | StoreComposer *StoreComposer 17 | // MaxSize defines how many bytes may be stored in one single upload. If its 18 | // value is is 0 or smaller no limit will be enforced. 19 | MaxSize int64 20 | // BasePath defines the URL path used for handling uploads, e.g. "/files/". 21 | // If no trailing slash is presented it will be added. You may specify an 22 | // absolute URL containing a scheme, e.g. "http://tus.io" 23 | BasePath string 24 | isAbs bool 25 | // NotifyCompleteUploads indicates whether sending notifications about 26 | // completed uploads using the CompleteUploads channel should be enabled. 27 | NotifyCompleteUploads bool 28 | // NotifyTerminatedUploads indicates whether sending notifications about 29 | // terminated uploads using the TerminatedUploads channel should be enabled. 30 | NotifyTerminatedUploads bool 31 | // NotifyUploadProgress indicates whether sending notifications about 32 | // the upload progress using the UploadProgress channel should be enabled. 33 | NotifyUploadProgress bool 34 | // NotifyCreatedUploads indicates whether sending notifications about 35 | // the upload having been created using the CreatedUploads channel should be enabled. 36 | NotifyCreatedUploads bool 37 | // Logger is the logger to use internally, mostly for printing requests. 38 | Logger *log.Logger 39 | // Respect the X-Forwarded-Host, X-Forwarded-Proto and Forwarded headers 40 | // potentially set by proxies when generating an absolute URL in the 41 | // response to POST requests. 42 | RespectForwardedHeaders bool 43 | // PreUploadreateCCallback will be invoked before a new upload is created, if the 44 | // property is supplied. If the callback returns nil, the upload will be created. 45 | // Otherwise the HTTP request will be aborted. This can be used to implement 46 | // validation of upload metadata etc. 47 | PreUploadCreateCallback func(hook HookEvent) error 48 | } 49 | 50 | func (config *Config) validate() error { 51 | if config.Logger == nil { 52 | config.Logger = log.New(os.Stdout, "[tusd] ", log.Ldate|log.Ltime) 53 | } 54 | 55 | base := config.BasePath 56 | uri, err := url.Parse(base) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | // Ensure base path ends with slash to remove logic from absFileURL 62 | if base != "" && string(base[len(base)-1]) != "/" { 63 | base += "/" 64 | } 65 | 66 | // Ensure base path begins with slash if not absolute (starts with scheme) 67 | if !uri.IsAbs() && len(base) > 0 && string(base[0]) != "/" { 68 | base = "/" + base 69 | } 70 | config.BasePath = base 71 | config.isAbs = uri.IsAbs() 72 | 73 | if config.StoreComposer == nil { 74 | return errors.New("tusd: StoreComposer must no be nil") 75 | } 76 | 77 | if config.StoreComposer.Core == nil { 78 | return errors.New("tusd: StoreComposer in Config needs to contain a non-nil core") 79 | } 80 | 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /vendor/github.com/tus/tusd/pkg/handler/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package handler provides ways to accept tus 1.0 calls using HTTP. 3 | 4 | tus is a protocol based on HTTP for resumable file uploads. Resumable means that 5 | an upload can be interrupted at any moment and can be resumed without 6 | re-uploading the previous data again. An interruption may happen willingly, if 7 | the user wants to pause, or by accident in case of an network issue or server 8 | outage (http://tus.io). 9 | 10 | The basics of tusd 11 | 12 | tusd was designed in way which allows an flexible and customizable usage. We 13 | wanted to avoid binding this package to a specific storage system – particularly 14 | a proprietary third-party software. Therefore tusd is an abstract layer whose 15 | only job is to accept incoming HTTP requests, validate them according to the 16 | specification and finally passes them to the data store. 17 | 18 | The data store is another important component in tusd's architecture whose 19 | purpose is to do the actual file handling. It has to write the incoming upload 20 | to a persistent storage system and retrieve information about an upload's 21 | current state. Therefore it is the only part of the system which communicates 22 | directly with the underlying storage system, whether it be the local disk, a 23 | remote FTP server or cloud providers such as AWS S3. 24 | 25 | Using a store composer 26 | 27 | The only hard requirements for a data store can be found in the DataStore 28 | interface. It contains methods for creating uploads (NewUpload), writing to 29 | them (WriteChunk) and retrieving their status (GetInfo). However, there 30 | are many more features which are not mandatory but may still be used. 31 | These are contained in their own interfaces which all share the *DataStore 32 | suffix. For example, GetReaderDataStore which enables downloading uploads or 33 | TerminaterDataStore which allows uploads to be terminated. 34 | 35 | The store composer offers a way to combine the basic data store - the core - 36 | implementation and these additional extensions: 37 | 38 | composer := tusd.NewStoreComposer() 39 | composer.UseCore(dataStore) // Implements DataStore 40 | composer.UseTerminater(terminater) // Implements TerminaterDataStore 41 | composer.UseLocker(locker) // Implements LockerDataStore 42 | 43 | The corresponding methods for adding an extension to the composer are prefixed 44 | with Use* followed by the name of the corresponding interface. However, most 45 | data store provide multiple extensions and adding all of them manually can be 46 | tedious and error-prone. Therefore, all data store distributed with tusd provide 47 | an UseIn() method which does this job automatically. For example, this is the 48 | S3 store in action (see S3Store.UseIn): 49 | 50 | store := s3store.New(…) 51 | locker := memorylocker.New() 52 | composer := tusd.NewStoreComposer() 53 | store.UseIn(composer) 54 | locker.UseIn(composer) 55 | 56 | Finally, once you are done with composing your data store, you can pass it 57 | inside the Config struct in order to create create a new tusd HTTP handler: 58 | 59 | config := tusd.Config{ 60 | StoreComposer: composer, 61 | BasePath: "/files/", 62 | } 63 | handler, err := tusd.NewHandler(config) 64 | 65 | This handler can then be mounted to a specific path, e.g. /files: 66 | 67 | http.Handle("/files/", http.StripPrefix("/files/", handler)) 68 | */ 69 | package handler 70 | -------------------------------------------------------------------------------- /vendor/github.com/tus/tusd/pkg/handler/handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/bmizerany/pat" 7 | ) 8 | 9 | // Handler is a ready to use handler with routing (using pat) 10 | type Handler struct { 11 | *UnroutedHandler 12 | http.Handler 13 | } 14 | 15 | // NewHandler creates a routed tus protocol handler. This is the simplest 16 | // way to use tusd but may not be as configurable as you require. If you are 17 | // integrating this into an existing app you may like to use tusd.NewUnroutedHandler 18 | // instead. Using tusd.NewUnroutedHandler allows the tus handlers to be combined into 19 | // your existing router (aka mux) directly. It also allows the GET and DELETE 20 | // endpoints to be customized. These are not part of the protocol so can be 21 | // changed depending on your needs. 22 | func NewHandler(config Config) (*Handler, error) { 23 | if err := config.validate(); err != nil { 24 | return nil, err 25 | } 26 | 27 | handler, err := NewUnroutedHandler(config) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | routedHandler := &Handler{ 33 | UnroutedHandler: handler, 34 | } 35 | 36 | mux := pat.New() 37 | 38 | routedHandler.Handler = handler.Middleware(mux) 39 | 40 | mux.Post("", http.HandlerFunc(handler.PostFile)) 41 | mux.Head(":id", http.HandlerFunc(handler.HeadFile)) 42 | mux.Add("PATCH", ":id", http.HandlerFunc(handler.PatchFile)) 43 | mux.Get(":id", http.HandlerFunc(handler.GetFile)) 44 | 45 | // Only attach the DELETE handler if the Terminate() method is provided 46 | if config.StoreComposer.UsesTerminater { 47 | mux.Del(":id", http.HandlerFunc(handler.DelFile)) 48 | } 49 | 50 | return routedHandler, nil 51 | } 52 | -------------------------------------------------------------------------------- /vendor/github.com/tus/tusd/pkg/handler/log.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "log" 5 | ) 6 | 7 | func (h *UnroutedHandler) log(eventName string, details ...string) { 8 | LogEvent(h.logger, eventName, details...) 9 | } 10 | 11 | func LogEvent(logger *log.Logger, eventName string, details ...string) { 12 | result := make([]byte, 0, 100) 13 | 14 | result = append(result, `event="`...) 15 | result = append(result, eventName...) 16 | result = append(result, `" `...) 17 | 18 | for i := 0; i < len(details); i += 2 { 19 | result = append(result, details[i]...) 20 | result = append(result, `="`...) 21 | result = append(result, details[i+1]...) 22 | result = append(result, `" `...) 23 | } 24 | 25 | result = append(result, "\n"...) 26 | logger.Output(2, string(result)) 27 | } 28 | -------------------------------------------------------------------------------- /vendor/github.com/tus/tusd/pkg/handler/metrics.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | "sync/atomic" 7 | ) 8 | 9 | // Metrics provides numbers about the usage of the tusd handler. Since these may 10 | // be accessed from multiple goroutines, it is necessary to read and modify them 11 | // atomically using the functions exposed in the sync/atomic package, such as 12 | // atomic.LoadUint64. In addition the maps must not be modified to prevent data 13 | // races. 14 | type Metrics struct { 15 | // RequestTotal counts the number of incoming requests per method 16 | RequestsTotal map[string]*uint64 17 | // ErrorsTotal counts the number of returned errors by their message 18 | ErrorsTotal *ErrorsTotalMap 19 | BytesReceived *uint64 20 | UploadsFinished *uint64 21 | UploadsCreated *uint64 22 | UploadsTerminated *uint64 23 | } 24 | 25 | // incRequestsTotal increases the counter for this request method atomically by 26 | // one. The method must be one of GET, HEAD, POST, PATCH, DELETE. 27 | func (m Metrics) incRequestsTotal(method string) { 28 | if ptr, ok := m.RequestsTotal[method]; ok { 29 | atomic.AddUint64(ptr, 1) 30 | } 31 | } 32 | 33 | // incErrorsTotal increases the counter for this error atomically by one. 34 | func (m Metrics) incErrorsTotal(err HTTPError) { 35 | ptr := m.ErrorsTotal.retrievePointerFor(err) 36 | atomic.AddUint64(ptr, 1) 37 | } 38 | 39 | // incBytesReceived increases the number of received bytes atomically be the 40 | // specified number. 41 | func (m Metrics) incBytesReceived(delta uint64) { 42 | atomic.AddUint64(m.BytesReceived, delta) 43 | } 44 | 45 | // incUploadsFinished increases the counter for finished uploads atomically by one. 46 | func (m Metrics) incUploadsFinished() { 47 | atomic.AddUint64(m.UploadsFinished, 1) 48 | } 49 | 50 | // incUploadsCreated increases the counter for completed uploads atomically by one. 51 | func (m Metrics) incUploadsCreated() { 52 | atomic.AddUint64(m.UploadsCreated, 1) 53 | } 54 | 55 | // incUploadsTerminated increases the counter for completed uploads atomically by one. 56 | func (m Metrics) incUploadsTerminated() { 57 | atomic.AddUint64(m.UploadsTerminated, 1) 58 | } 59 | 60 | func newMetrics() Metrics { 61 | return Metrics{ 62 | RequestsTotal: map[string]*uint64{ 63 | "GET": new(uint64), 64 | "HEAD": new(uint64), 65 | "POST": new(uint64), 66 | "PATCH": new(uint64), 67 | "DELETE": new(uint64), 68 | "OPTIONS": new(uint64), 69 | }, 70 | ErrorsTotal: newErrorsTotalMap(), 71 | BytesReceived: new(uint64), 72 | UploadsFinished: new(uint64), 73 | UploadsCreated: new(uint64), 74 | UploadsTerminated: new(uint64), 75 | } 76 | } 77 | 78 | // ErrorsTotalMap stores the counters for the different HTTP errors. 79 | type ErrorsTotalMap struct { 80 | lock sync.RWMutex 81 | counter map[simpleHTTPError]*uint64 82 | } 83 | 84 | type simpleHTTPError struct { 85 | Message string 86 | StatusCode int 87 | } 88 | 89 | func simplifyHTTPError(err HTTPError) simpleHTTPError { 90 | return simpleHTTPError{ 91 | Message: err.Error(), 92 | StatusCode: err.StatusCode(), 93 | } 94 | } 95 | 96 | func newErrorsTotalMap() *ErrorsTotalMap { 97 | m := make(map[simpleHTTPError]*uint64, 20) 98 | return &ErrorsTotalMap{ 99 | counter: m, 100 | } 101 | } 102 | 103 | // retrievePointerFor returns (after creating it if necessary) the pointer to 104 | // the counter for the error. 105 | func (e *ErrorsTotalMap) retrievePointerFor(err HTTPError) *uint64 { 106 | serr := simplifyHTTPError(err) 107 | e.lock.RLock() 108 | ptr, ok := e.counter[serr] 109 | e.lock.RUnlock() 110 | if ok { 111 | return ptr 112 | } 113 | 114 | // For pointer creation, a write-lock is required 115 | e.lock.Lock() 116 | // We ensure that the pointer wasn't created in the meantime 117 | if ptr, ok = e.counter[serr]; !ok { 118 | ptr = new(uint64) 119 | e.counter[serr] = ptr 120 | } 121 | e.lock.Unlock() 122 | 123 | return ptr 124 | } 125 | 126 | // Load retrieves the map of the counter pointers atomically 127 | func (e *ErrorsTotalMap) Load() map[HTTPError]*uint64 { 128 | m := make(map[HTTPError]*uint64, len(e.counter)) 129 | e.lock.RLock() 130 | for err, ptr := range e.counter { 131 | httpErr := NewHTTPError(errors.New(err.Message), err.StatusCode) 132 | m[httpErr] = ptr 133 | } 134 | e.lock.RUnlock() 135 | 136 | return m 137 | } 138 | -------------------------------------------------------------------------------- /vendor/gopkg.in/yaml.v2/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.4 5 | - 1.5 6 | - 1.6 7 | - 1.7 8 | - 1.8 9 | - 1.9 10 | - tip 11 | 12 | go_import_path: gopkg.in/yaml.v2 13 | -------------------------------------------------------------------------------- /vendor/gopkg.in/yaml.v2/LICENSE.libyaml: -------------------------------------------------------------------------------- 1 | The following files were ported to Go from C files of libyaml, and thus 2 | are still covered by their original copyright and license: 3 | 4 | apic.go 5 | emitterc.go 6 | parserc.go 7 | readerc.go 8 | scannerc.go 9 | writerc.go 10 | yamlh.go 11 | yamlprivateh.go 12 | 13 | Copyright (c) 2006 Kirill Simonov 14 | 15 | Permission is hereby granted, free of charge, to any person obtaining a copy of 16 | this software and associated documentation files (the "Software"), to deal in 17 | the Software without restriction, including without limitation the rights to 18 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 19 | of the Software, and to permit persons to whom the Software is furnished to do 20 | so, subject to the following conditions: 21 | 22 | The above copyright notice and this permission notice shall be included in all 23 | copies or substantial portions of the Software. 24 | 25 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 26 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 27 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 28 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 29 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 30 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 | SOFTWARE. 32 | -------------------------------------------------------------------------------- /vendor/gopkg.in/yaml.v2/NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2011-2016 Canonical Ltd. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /vendor/gopkg.in/yaml.v2/README.md: -------------------------------------------------------------------------------- 1 | # YAML support for the Go language 2 | 3 | Introduction 4 | ------------ 5 | 6 | The yaml package enables Go programs to comfortably encode and decode YAML 7 | values. It was developed within [Canonical](https://www.canonical.com) as 8 | part of the [juju](https://juju.ubuntu.com) project, and is based on a 9 | pure Go port of the well-known [libyaml](http://pyyaml.org/wiki/LibYAML) 10 | C library to parse and generate YAML data quickly and reliably. 11 | 12 | Compatibility 13 | ------------- 14 | 15 | The yaml package supports most of YAML 1.1 and 1.2, including support for 16 | anchors, tags, map merging, etc. Multi-document unmarshalling is not yet 17 | implemented, and base-60 floats from YAML 1.1 are purposefully not 18 | supported since they're a poor design and are gone in YAML 1.2. 19 | 20 | Installation and usage 21 | ---------------------- 22 | 23 | The import path for the package is *gopkg.in/yaml.v2*. 24 | 25 | To install it, run: 26 | 27 | go get gopkg.in/yaml.v2 28 | 29 | API documentation 30 | ----------------- 31 | 32 | If opened in a browser, the import path itself leads to the API documentation: 33 | 34 | * [https://gopkg.in/yaml.v2](https://gopkg.in/yaml.v2) 35 | 36 | API stability 37 | ------------- 38 | 39 | The package API for yaml v2 will remain stable as described in [gopkg.in](https://gopkg.in). 40 | 41 | 42 | License 43 | ------- 44 | 45 | The yaml package is licensed under the Apache License 2.0. Please see the LICENSE file for details. 46 | 47 | 48 | Example 49 | ------- 50 | 51 | ```Go 52 | package main 53 | 54 | import ( 55 | "fmt" 56 | "log" 57 | 58 | "gopkg.in/yaml.v2" 59 | ) 60 | 61 | var data = ` 62 | a: Easy! 63 | b: 64 | c: 2 65 | d: [3, 4] 66 | ` 67 | 68 | // Note: struct fields must be public in order for unmarshal to 69 | // correctly populate the data. 70 | type T struct { 71 | A string 72 | B struct { 73 | RenamedC int `yaml:"c"` 74 | D []int `yaml:",flow"` 75 | } 76 | } 77 | 78 | func main() { 79 | t := T{} 80 | 81 | err := yaml.Unmarshal([]byte(data), &t) 82 | if err != nil { 83 | log.Fatalf("error: %v", err) 84 | } 85 | fmt.Printf("--- t:\n%v\n\n", t) 86 | 87 | d, err := yaml.Marshal(&t) 88 | if err != nil { 89 | log.Fatalf("error: %v", err) 90 | } 91 | fmt.Printf("--- t dump:\n%s\n\n", string(d)) 92 | 93 | m := make(map[interface{}]interface{}) 94 | 95 | err = yaml.Unmarshal([]byte(data), &m) 96 | if err != nil { 97 | log.Fatalf("error: %v", err) 98 | } 99 | fmt.Printf("--- m:\n%v\n\n", m) 100 | 101 | d, err = yaml.Marshal(&m) 102 | if err != nil { 103 | log.Fatalf("error: %v", err) 104 | } 105 | fmt.Printf("--- m dump:\n%s\n\n", string(d)) 106 | } 107 | ``` 108 | 109 | This example will generate the following output: 110 | 111 | ``` 112 | --- t: 113 | {Easy! {2 [3 4]}} 114 | 115 | --- t dump: 116 | a: Easy! 117 | b: 118 | c: 2 119 | d: [3, 4] 120 | 121 | 122 | --- m: 123 | map[a:Easy! b:map[c:2 d:[3 4]]] 124 | 125 | --- m dump: 126 | a: Easy! 127 | b: 128 | c: 2 129 | d: 130 | - 3 131 | - 4 132 | ``` 133 | 134 | -------------------------------------------------------------------------------- /vendor/gopkg.in/yaml.v2/go.mod: -------------------------------------------------------------------------------- 1 | module "gopkg.in/yaml.v2" 2 | 3 | require ( 4 | "gopkg.in/check.v1" v0.0.0-20161208181325-20d25e280405 5 | ) 6 | -------------------------------------------------------------------------------- /vendor/gopkg.in/yaml.v2/sorter.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | import ( 4 | "reflect" 5 | "unicode" 6 | ) 7 | 8 | type keyList []reflect.Value 9 | 10 | func (l keyList) Len() int { return len(l) } 11 | func (l keyList) Swap(i, j int) { l[i], l[j] = l[j], l[i] } 12 | func (l keyList) Less(i, j int) bool { 13 | a := l[i] 14 | b := l[j] 15 | ak := a.Kind() 16 | bk := b.Kind() 17 | for (ak == reflect.Interface || ak == reflect.Ptr) && !a.IsNil() { 18 | a = a.Elem() 19 | ak = a.Kind() 20 | } 21 | for (bk == reflect.Interface || bk == reflect.Ptr) && !b.IsNil() { 22 | b = b.Elem() 23 | bk = b.Kind() 24 | } 25 | af, aok := keyFloat(a) 26 | bf, bok := keyFloat(b) 27 | if aok && bok { 28 | if af != bf { 29 | return af < bf 30 | } 31 | if ak != bk { 32 | return ak < bk 33 | } 34 | return numLess(a, b) 35 | } 36 | if ak != reflect.String || bk != reflect.String { 37 | return ak < bk 38 | } 39 | ar, br := []rune(a.String()), []rune(b.String()) 40 | for i := 0; i < len(ar) && i < len(br); i++ { 41 | if ar[i] == br[i] { 42 | continue 43 | } 44 | al := unicode.IsLetter(ar[i]) 45 | bl := unicode.IsLetter(br[i]) 46 | if al && bl { 47 | return ar[i] < br[i] 48 | } 49 | if al || bl { 50 | return bl 51 | } 52 | var ai, bi int 53 | var an, bn int64 54 | if ar[i] == '0' || br[i] == '0' { 55 | for j := i-1; j >= 0 && unicode.IsDigit(ar[j]); j-- { 56 | if ar[j] != '0' { 57 | an = 1 58 | bn = 1 59 | break 60 | } 61 | } 62 | } 63 | for ai = i; ai < len(ar) && unicode.IsDigit(ar[ai]); ai++ { 64 | an = an*10 + int64(ar[ai]-'0') 65 | } 66 | for bi = i; bi < len(br) && unicode.IsDigit(br[bi]); bi++ { 67 | bn = bn*10 + int64(br[bi]-'0') 68 | } 69 | if an != bn { 70 | return an < bn 71 | } 72 | if ai != bi { 73 | return ai < bi 74 | } 75 | return ar[i] < br[i] 76 | } 77 | return len(ar) < len(br) 78 | } 79 | 80 | // keyFloat returns a float value for v if it is a number/bool 81 | // and whether it is a number/bool or not. 82 | func keyFloat(v reflect.Value) (f float64, ok bool) { 83 | switch v.Kind() { 84 | case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 85 | return float64(v.Int()), true 86 | case reflect.Float32, reflect.Float64: 87 | return v.Float(), true 88 | case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: 89 | return float64(v.Uint()), true 90 | case reflect.Bool: 91 | if v.Bool() { 92 | return 1, true 93 | } 94 | return 0, true 95 | } 96 | return 0, false 97 | } 98 | 99 | // numLess returns whether a < b. 100 | // a and b must necessarily have the same kind. 101 | func numLess(a, b reflect.Value) bool { 102 | switch a.Kind() { 103 | case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 104 | return a.Int() < b.Int() 105 | case reflect.Float32, reflect.Float64: 106 | return a.Float() < b.Float() 107 | case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: 108 | return a.Uint() < b.Uint() 109 | case reflect.Bool: 110 | return !a.Bool() && b.Bool() 111 | } 112 | panic("not a number") 113 | } 114 | -------------------------------------------------------------------------------- /vendor/gopkg.in/yaml.v2/writerc.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | // Set the writer error and return false. 4 | func yaml_emitter_set_writer_error(emitter *yaml_emitter_t, problem string) bool { 5 | emitter.error = yaml_WRITER_ERROR 6 | emitter.problem = problem 7 | return false 8 | } 9 | 10 | // Flush the output buffer. 11 | func yaml_emitter_flush(emitter *yaml_emitter_t) bool { 12 | if emitter.write_handler == nil { 13 | panic("write handler not set") 14 | } 15 | 16 | // Check if the buffer is empty. 17 | if emitter.buffer_pos == 0 { 18 | return true 19 | } 20 | 21 | if err := emitter.write_handler(emitter, emitter.buffer[:emitter.buffer_pos]); err != nil { 22 | return yaml_emitter_set_writer_error(emitter, "write error: "+err.Error()) 23 | } 24 | emitter.buffer_pos = 0 25 | return true 26 | } 27 | -------------------------------------------------------------------------------- /vendor/gopkg.in/yaml.v2/yamlprivateh.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | const ( 4 | // The size of the input raw buffer. 5 | input_raw_buffer_size = 512 6 | 7 | // The size of the input buffer. 8 | // It should be possible to decode the whole raw buffer. 9 | input_buffer_size = input_raw_buffer_size * 3 10 | 11 | // The size of the output buffer. 12 | output_buffer_size = 128 13 | 14 | // The size of the output raw buffer. 15 | // It should be possible to encode the whole output buffer. 16 | output_raw_buffer_size = (output_buffer_size*2 + 2) 17 | 18 | // The size of other stacks and queues. 19 | initial_stack_size = 16 20 | initial_queue_size = 16 21 | initial_string_size = 16 22 | ) 23 | 24 | // Check if the character at the specified position is an alphabetical 25 | // character, a digit, '_', or '-'. 26 | func is_alpha(b []byte, i int) bool { 27 | return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'Z' || b[i] >= 'a' && b[i] <= 'z' || b[i] == '_' || b[i] == '-' 28 | } 29 | 30 | // Check if the character at the specified position is a digit. 31 | func is_digit(b []byte, i int) bool { 32 | return b[i] >= '0' && b[i] <= '9' 33 | } 34 | 35 | // Get the value of a digit. 36 | func as_digit(b []byte, i int) int { 37 | return int(b[i]) - '0' 38 | } 39 | 40 | // Check if the character at the specified position is a hex-digit. 41 | func is_hex(b []byte, i int) bool { 42 | return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'F' || b[i] >= 'a' && b[i] <= 'f' 43 | } 44 | 45 | // Get the value of a hex-digit. 46 | func as_hex(b []byte, i int) int { 47 | bi := b[i] 48 | if bi >= 'A' && bi <= 'F' { 49 | return int(bi) - 'A' + 10 50 | } 51 | if bi >= 'a' && bi <= 'f' { 52 | return int(bi) - 'a' + 10 53 | } 54 | return int(bi) - '0' 55 | } 56 | 57 | // Check if the character is ASCII. 58 | func is_ascii(b []byte, i int) bool { 59 | return b[i] <= 0x7F 60 | } 61 | 62 | // Check if the character at the start of the buffer can be printed unescaped. 63 | func is_printable(b []byte, i int) bool { 64 | return ((b[i] == 0x0A) || // . == #x0A 65 | (b[i] >= 0x20 && b[i] <= 0x7E) || // #x20 <= . <= #x7E 66 | (b[i] == 0xC2 && b[i+1] >= 0xA0) || // #0xA0 <= . <= #xD7FF 67 | (b[i] > 0xC2 && b[i] < 0xED) || 68 | (b[i] == 0xED && b[i+1] < 0xA0) || 69 | (b[i] == 0xEE) || 70 | (b[i] == 0xEF && // #xE000 <= . <= #xFFFD 71 | !(b[i+1] == 0xBB && b[i+2] == 0xBF) && // && . != #xFEFF 72 | !(b[i+1] == 0xBF && (b[i+2] == 0xBE || b[i+2] == 0xBF)))) 73 | } 74 | 75 | // Check if the character at the specified position is NUL. 76 | func is_z(b []byte, i int) bool { 77 | return b[i] == 0x00 78 | } 79 | 80 | // Check if the beginning of the buffer is a BOM. 81 | func is_bom(b []byte, i int) bool { 82 | return b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF 83 | } 84 | 85 | // Check if the character at the specified position is space. 86 | func is_space(b []byte, i int) bool { 87 | return b[i] == ' ' 88 | } 89 | 90 | // Check if the character at the specified position is tab. 91 | func is_tab(b []byte, i int) bool { 92 | return b[i] == '\t' 93 | } 94 | 95 | // Check if the character at the specified position is blank (space or tab). 96 | func is_blank(b []byte, i int) bool { 97 | //return is_space(b, i) || is_tab(b, i) 98 | return b[i] == ' ' || b[i] == '\t' 99 | } 100 | 101 | // Check if the character at the specified position is a line break. 102 | func is_break(b []byte, i int) bool { 103 | return (b[i] == '\r' || // CR (#xD) 104 | b[i] == '\n' || // LF (#xA) 105 | b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) 106 | b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) 107 | b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9) // PS (#x2029) 108 | } 109 | 110 | func is_crlf(b []byte, i int) bool { 111 | return b[i] == '\r' && b[i+1] == '\n' 112 | } 113 | 114 | // Check if the character is a line break or NUL. 115 | func is_breakz(b []byte, i int) bool { 116 | //return is_break(b, i) || is_z(b, i) 117 | return ( // is_break: 118 | b[i] == '\r' || // CR (#xD) 119 | b[i] == '\n' || // LF (#xA) 120 | b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) 121 | b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) 122 | b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) 123 | // is_z: 124 | b[i] == 0) 125 | } 126 | 127 | // Check if the character is a line break, space, or NUL. 128 | func is_spacez(b []byte, i int) bool { 129 | //return is_space(b, i) || is_breakz(b, i) 130 | return ( // is_space: 131 | b[i] == ' ' || 132 | // is_breakz: 133 | b[i] == '\r' || // CR (#xD) 134 | b[i] == '\n' || // LF (#xA) 135 | b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) 136 | b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) 137 | b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) 138 | b[i] == 0) 139 | } 140 | 141 | // Check if the character is a line break, space, tab, or NUL. 142 | func is_blankz(b []byte, i int) bool { 143 | //return is_blank(b, i) || is_breakz(b, i) 144 | return ( // is_blank: 145 | b[i] == ' ' || b[i] == '\t' || 146 | // is_breakz: 147 | b[i] == '\r' || // CR (#xD) 148 | b[i] == '\n' || // LF (#xA) 149 | b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) 150 | b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) 151 | b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) 152 | b[i] == 0) 153 | } 154 | 155 | // Determine the width of the character. 156 | func width(b byte) int { 157 | // Don't replace these by a switch without first 158 | // confirming that it is being inlined. 159 | if b&0x80 == 0x00 { 160 | return 1 161 | } 162 | if b&0xE0 == 0xC0 { 163 | return 2 164 | } 165 | if b&0xF0 == 0xE0 { 166 | return 3 167 | } 168 | if b&0xF8 == 0xF0 { 169 | return 4 170 | } 171 | return 0 172 | 173 | } 174 | -------------------------------------------------------------------------------- /vendor/modules.txt: -------------------------------------------------------------------------------- 1 | # github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 2 | github.com/bmizerany/pat 3 | # github.com/davecgh/go-spew v1.1.1 4 | github.com/davecgh/go-spew/spew 5 | # github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db 6 | github.com/golang/snappy 7 | # github.com/pmezard/go-difflib v1.0.0 8 | github.com/pmezard/go-difflib/difflib 9 | # github.com/stretchr/testify v1.5.1 10 | github.com/stretchr/testify/assert 11 | github.com/stretchr/testify/require 12 | github.com/stretchr/testify/suite 13 | # github.com/syndtr/goleveldb v1.0.0 14 | github.com/syndtr/goleveldb/leveldb 15 | github.com/syndtr/goleveldb/leveldb/cache 16 | github.com/syndtr/goleveldb/leveldb/comparer 17 | github.com/syndtr/goleveldb/leveldb/errors 18 | github.com/syndtr/goleveldb/leveldb/filter 19 | github.com/syndtr/goleveldb/leveldb/iterator 20 | github.com/syndtr/goleveldb/leveldb/journal 21 | github.com/syndtr/goleveldb/leveldb/memdb 22 | github.com/syndtr/goleveldb/leveldb/opt 23 | github.com/syndtr/goleveldb/leveldb/storage 24 | github.com/syndtr/goleveldb/leveldb/table 25 | github.com/syndtr/goleveldb/leveldb/util 26 | # github.com/tus/tusd v1.1.0 27 | github.com/tus/tusd/internal/uid 28 | github.com/tus/tusd/pkg/filestore 29 | github.com/tus/tusd/pkg/handler 30 | # gopkg.in/yaml.v2 v2.2.2 31 | gopkg.in/yaml.v2 32 | --------------------------------------------------------------------------------