├── store ├── cache │ ├── doc.go │ ├── keyvalue.go │ ├── item.go │ ├── janitor.go │ └── cache.go ├── disk │ ├── doc.go │ ├── disk.go │ ├── pebble_386_arm.go │ ├── pogreb_windows_arm.go │ ├── error.go │ ├── kv.go │ ├── util_test.go │ ├── kv_test.go │ ├── buntdb.go │ ├── bboltdb.go │ ├── pebble_all.go │ ├── pogreb_all.go │ └── leveldb.go └── hybrid │ ├── memoryguard.go │ └── hybrid.go ├── filekv ├── errors.go ├── strategy.go ├── options.go ├── file_test.go ├── merge.go └── file.go ├── .github ├── workflows │ ├── lint-test.yml │ ├── build-test.yml │ ├── codeql-analysis.yml │ └── release-tag.yml └── dependabot.yml ├── LICENSE.md ├── cmd ├── filekv │ └── filekv.go └── example │ └── example.go ├── README.md ├── go.mod └── go.sum /store/cache/doc.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // Forked and adapted from https://github.com/patrickmn/go-cache 4 | -------------------------------------------------------------------------------- /store/cache/keyvalue.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type keyAndValue struct { 4 | key string 5 | value interface{} 6 | } 7 | -------------------------------------------------------------------------------- /store/disk/doc.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | // Forked ad adapted from https://github.com/alash3al/redix/blob/master/kvstore/leveldb/leveldb.go 4 | -------------------------------------------------------------------------------- /store/disk/disk.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | const Megabyte = 1 << 20 4 | 5 | var ( 6 | OpenPogrebDB func(string) (DB, error) 7 | OpenPebbleDB func(string) (DB, error) 8 | ) 9 | -------------------------------------------------------------------------------- /filekv/errors.go: -------------------------------------------------------------------------------- 1 | package filekv 2 | 3 | import "errors" 4 | 5 | var ( 6 | ErrItemExists = errors.New("item already exist") 7 | ErrItemFiltered = errors.New("item filtered") 8 | ) 9 | -------------------------------------------------------------------------------- /store/disk/pebble_386_arm.go: -------------------------------------------------------------------------------- 1 | //go:build 386 || arm 2 | 3 | package disk 4 | 5 | func init() { 6 | OpenPebbleDB = func(_ string) (DB, error) { 7 | return nil, ErrNotSupported 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /store/disk/pogreb_windows_arm.go: -------------------------------------------------------------------------------- 1 | //go:build (arm || arm64) && windows 2 | 3 | package disk 4 | 5 | import ( 6 | "bytes" 7 | "strconv" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | func init() { 13 | OpenPogrebDB = func(_ string) (DB, error) { 14 | return nil, ErrNotSupported 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /store/disk/error.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import "github.com/pkg/errors" 4 | 5 | var ( 6 | ErrNotImplemented = errors.New("not implemented") 7 | ErrNotFound = errors.New("not found") 8 | ErrNoData = errors.New("no data") 9 | ErrNotSupported = errors.New("not supported") 10 | ) 11 | -------------------------------------------------------------------------------- /store/cache/item.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "time" 4 | 5 | type Item struct { 6 | Object interface{} 7 | Expiration int64 8 | } 9 | 10 | func (item Item) Expired() bool { 11 | if item.Expiration == 0 { 12 | return false 13 | } 14 | return time.Now().UnixNano() > item.Expiration 15 | } 16 | -------------------------------------------------------------------------------- /.github/workflows/lint-test.yml: -------------------------------------------------------------------------------- 1 | name: 🙏🏻 Lint Test 2 | on: 3 | pull_request: 4 | workflow_dispatch: 5 | 6 | jobs: 7 | lint: 8 | name: Lint Test 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout code 12 | uses: actions/checkout@v3 13 | - name: Run golangci-lint 14 | uses: golangci/golangci-lint-action@v3 15 | with: 16 | version: latest 17 | args: --timeout 5m 18 | working-directory: . -------------------------------------------------------------------------------- /filekv/strategy.go: -------------------------------------------------------------------------------- 1 | package filekv 2 | 3 | type Strategy uint8 4 | 5 | const ( 6 | None Strategy = iota 7 | // Uses go standard map without eviction - grows linearly with the items number 8 | MemoryMap 9 | // MemoryLRU keeps in memory the last x items and filter with a look-back probabilistic window with fixed size 10 | MemoryLRU 11 | // MemoryFilter uses bitset in-memory filters to remove duplicates 12 | MemoryFilter 13 | // Use full disk kv store to remove all duplicates - it should have low heap memory footprint but lots of I/O interactions 14 | DiskFilter 15 | ) 16 | -------------------------------------------------------------------------------- /store/cache/janitor.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type janitor struct { 8 | Interval time.Duration 9 | stop chan struct{} 10 | } 11 | 12 | func (j *janitor) Run(c *cacheMemory) { 13 | ticker := time.NewTicker(j.Interval) 14 | for { 15 | select { 16 | case <-ticker.C: 17 | c.DeleteExpired() 18 | case <-j.stop: 19 | ticker.Stop() 20 | return 21 | } 22 | } 23 | } 24 | 25 | func stopJanitor(c *cacheMemory) { 26 | c.janitor.stop <- struct{}{} 27 | } 28 | 29 | func runJanitor(c *cacheMemory, ci time.Duration) { 30 | j := &janitor{ 31 | Interval: ci, 32 | stop: make(chan struct{}), 33 | } 34 | c.janitor = j 35 | go j.Run(c) 36 | } 37 | -------------------------------------------------------------------------------- /store/hybrid/memoryguard.go: -------------------------------------------------------------------------------- 1 | package hybrid 2 | 3 | import "time" 4 | 5 | type memoryguard struct { 6 | Enabled bool 7 | Interval time.Duration 8 | stop chan bool 9 | } 10 | 11 | func (mg *memoryguard) Run(hm *HybridMap) { 12 | ticker := time.NewTicker(mg.Interval) 13 | for { 14 | select { 15 | case <-ticker.C: 16 | hm.TuneMemory() 17 | case <-mg.stop: 18 | ticker.Stop() 19 | return 20 | } 21 | } 22 | } 23 | 24 | func stopMemoryGuard(hm *HybridMap) { 25 | hm.memoryguard.stop <- true 26 | } 27 | 28 | func runMemoryGuard(c *HybridMap, ci time.Duration) { 29 | mg := &memoryguard{ 30 | Interval: ci, 31 | stop: make(chan bool), 32 | } 33 | c.memoryguard = mg 34 | go mg.Run(c) 35 | } 36 | -------------------------------------------------------------------------------- /filekv/options.go: -------------------------------------------------------------------------------- 1 | package filekv 2 | 3 | var ( 4 | BufferSize = 50 * 1024 * 1024 // 50Mb 5 | Separator = ";;;" 6 | NewLine = "\n" 7 | FpRatio = 0.0001 8 | MaxItems = uint(250000) 9 | ) 10 | 11 | type Options struct { 12 | Path string 13 | Compress bool 14 | MaxItems uint 15 | Cleanup bool 16 | SkipEmpty bool 17 | FilterCallback func(k, v []byte) bool 18 | Dedupe Strategy 19 | } 20 | 21 | type Stats struct { 22 | NumberOfFilteredItems uint 23 | NumberOfAddedItems uint 24 | NumberOfDupedItems uint 25 | NumberOfItems uint 26 | } 27 | 28 | var DefaultOptions Options = Options{ 29 | Compress: false, 30 | Cleanup: true, 31 | Dedupe: MemoryLRU, 32 | SkipEmpty: true, 33 | } 34 | -------------------------------------------------------------------------------- /.github/workflows/build-test.yml: -------------------------------------------------------------------------------- 1 | name: 🔨 Build Test 2 | on: 3 | pull_request: 4 | workflow_dispatch: 5 | 6 | 7 | jobs: 8 | build: 9 | name: Test Builds 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Set up Go 13 | uses: actions/setup-go@v3 14 | with: 15 | go-version: 1.18 16 | 17 | - name: Check out code 18 | uses: actions/checkout@v3 19 | 20 | - name: Test 21 | run: go test ./... 22 | 23 | - name: Test Hybrid Hmap 24 | run: go run . 25 | working-directory: cmd/example 26 | 27 | - name: Test FileKv 28 | run: go run . 29 | working-directory: cmd/filekv 30 | - name: Build 31 | run: go build ./cmd/example 32 | 33 | - name: Run example 34 | run: ./example 35 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: 🚨 CodeQL Analysis 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | branches: 7 | - main 8 | 9 | jobs: 10 | analyze: 11 | name: Analyze 12 | runs-on: ubuntu-latest 13 | permissions: 14 | actions: read 15 | contents: read 16 | security-events: write 17 | 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | language: [ 'go' ] 22 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 23 | 24 | steps: 25 | - name: Checkout repository 26 | uses: actions/checkout@v3 27 | 28 | # Initializes the CodeQL tools for scanning. 29 | - name: Initialize CodeQL 30 | uses: github/codeql-action/init@v2 31 | with: 32 | languages: ${{ matrix.language }} 33 | 34 | - name: Autobuild 35 | uses: github/codeql-action/autobuild@v2 36 | 37 | - name: Perform CodeQL Analysis 38 | uses: github/codeql-action/analyze@v2 -------------------------------------------------------------------------------- /store/disk/kv.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import "time" 4 | 5 | // DB Interface 6 | type DB interface { 7 | Incr(k string, by int64) (int64, error) 8 | Set(k string, v []byte, ttl time.Duration) error 9 | MSet(data map[string][]byte) error 10 | Get(k string) ([]byte, error) 11 | MGet(keys []string) [][]byte 12 | TTL(key string) int64 13 | MDel(keys []string) error 14 | Del(key string) error 15 | Scan(ScannerOpt ScannerOptions) error 16 | Size() int64 17 | GC() error 18 | Close() 19 | } 20 | 21 | // ScannerOptions - represents the options for a scanner 22 | type ScannerOptions struct { 23 | // from where to start 24 | Offset string 25 | 26 | // whether to include the value of the offset in the result or not 27 | IncludeOffset bool 28 | 29 | // the prefix that must be exists in each key in the iteration 30 | Prefix string 31 | 32 | // fetch the values (true) or this is a key only iteration (false) 33 | FetchValues bool 34 | 35 | // the handler that handles the incoming data 36 | Handler func(k []byte, v []byte) error 37 | } 38 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 ProjectDiscovery, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/release-tag.yml: -------------------------------------------------------------------------------- 1 | name: 🔖 Release Tag 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | - cron: '0 0 * * 0' 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Check out code 13 | uses: actions/checkout@v3 14 | with: 15 | fetch-depth: 0 16 | 17 | - name: Get Commit Count 18 | id: get_commit 19 | run: git rev-list `git rev-list --tags --no-walk --max-count=1`..HEAD --count | xargs -I {} echo COMMIT_COUNT={} >> $GITHUB_OUTPUT 20 | 21 | - name: Create release and tag 22 | if: ${{ steps.get_commit.outputs.COMMIT_COUNT > 0 }} 23 | id: tag_version 24 | uses: mathieudutour/github-tag-action@v6.1 25 | with: 26 | github_token: ${{ secrets.GITHUB_TOKEN }} 27 | 28 | - name: Create a GitHub release 29 | if: ${{ steps.get_commit.outputs.COMMIT_COUNT > 0 }} 30 | uses: actions/create-release@v1 31 | env: 32 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 33 | with: 34 | tag_name: ${{ steps.tag_version.outputs.new_tag }} 35 | release_name: Release ${{ steps.tag_version.outputs.new_tag }} 36 | body: ${{ steps.tag_version.outputs.changelog }} -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | 9 | # Maintain dependencies for GitHub Actions 10 | - package-ecosystem: "github-actions" 11 | directory: "/" 12 | schedule: 13 | interval: "weekly" 14 | target-branch: "main" 15 | commit-message: 16 | prefix: "chore" 17 | include: "scope" 18 | labels: 19 | - "Type: Maintenance" 20 | 21 | # Maintain dependencies for go modules 22 | - package-ecosystem: "gomod" 23 | directory: "/" 24 | schedule: 25 | interval: "weekly" 26 | target-branch: "main" 27 | commit-message: 28 | prefix: "chore" 29 | include: "scope" 30 | labels: 31 | - "Type: Maintenance" 32 | 33 | # Maintain dependencies for docker 34 | - package-ecosystem: "docker" 35 | directory: "/" 36 | schedule: 37 | interval: "weekly" 38 | target-branch: "main" 39 | commit-message: 40 | prefix: "chore" 41 | include: "scope" 42 | labels: 43 | - "Type: Maintenance" -------------------------------------------------------------------------------- /filekv/file_test.go: -------------------------------------------------------------------------------- 1 | package filekv 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/rs/xid" 10 | ) 11 | 12 | func TestFile(t *testing.T) { 13 | // add 200k items 14 | n := 100000 15 | var items1, items2 []string 16 | for i := 0; i < n; i++ { 17 | items1 = append(items1, xid.New().String()) 18 | items2 = append(items2, xid.New().String()) 19 | } 20 | 21 | dbFilename := filepath.Join(os.TempDir(), xid.New().String()) 22 | options := DefaultOptions 23 | options.Path = dbFilename 24 | fdb, err := Open(options) 25 | if err != nil { 26 | t.Error(err) 27 | } 28 | defer fdb.Close() 29 | 30 | _, err = fdb.Merge(items1, items2) 31 | if err != nil { 32 | t.Error(err) 33 | } 34 | 35 | err = fdb.Process() 36 | if err != nil { 37 | t.Error(err) 38 | } 39 | 40 | allItems := append(items1, items2...) 41 | // all items should already exist 42 | for _, item := range allItems { 43 | err := fdb.Set([]byte(item), nil) 44 | if err == nil { 45 | t.Errorf("item %s doesn't exist\n", err) 46 | } 47 | } 48 | 49 | count := 0 50 | err = fdb.Scan(func(k, v []byte) error { 51 | // items should respect the input order 52 | ks := string(k) 53 | if !strings.EqualFold(ks, allItems[count]) { 54 | t.Errorf("item %s doesn't respect order\n", ks) 55 | } 56 | count++ 57 | return nil 58 | }) 59 | if err != nil { 60 | t.Error(err) 61 | } 62 | expected := len(items1) + len(items2) 63 | if count != expected { 64 | t.Errorf("wrong number of items: wanted %d, got %d\n", expected, count) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /store/disk/util_test.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | type TestOperations struct { 12 | Set bool 13 | Get bool 14 | Scan bool 15 | Delete bool 16 | } 17 | 18 | func utiltestOperations(t *testing.T, db DB, maxItems int, operations TestOperations) { 19 | // set 20 | if operations.Set { 21 | for i := 0; i < maxItems; i++ { 22 | key := fmt.Sprint(i) 23 | value := []byte(key) 24 | if err := db.Set(key, value, time.Hour); err != nil { 25 | t.Error("[put] ", err) 26 | } 27 | } 28 | } 29 | 30 | // get 31 | if operations.Get { 32 | for i := 0; i < maxItems; i++ { 33 | key := fmt.Sprint(i) 34 | value := []byte(key) 35 | if data, err := db.Get(key); err != nil || !bytes.EqualFold(data, value) { 36 | t.Errorf("[get] got %s but wanted %s: err %s", string(data), string(value), err) 37 | } 38 | } 39 | } 40 | 41 | // scan 42 | if operations.Scan { 43 | read := 0 44 | err := db.Scan(ScannerOptions{ 45 | Handler: func(k, v []byte) error { 46 | _, _ = k, v 47 | read++ 48 | return nil 49 | }, 50 | }) 51 | if err != nil { 52 | t.Error(err) 53 | } 54 | if read != maxItems { 55 | t.Errorf("[scan] got %d but wanted %d", read, maxItems) 56 | } 57 | } 58 | 59 | // delete 60 | if operations.Delete { 61 | for i := 0; i < maxItems; i++ { 62 | key := fmt.Sprint(i) 63 | if err := db.Del(key); err != nil { 64 | t.Errorf("[del] couldn't delete %s err %s", key, err) 65 | } 66 | } 67 | } 68 | } 69 | 70 | func utiltestGetPath(t *testing.T) (string, error) { 71 | tmpdir, err := os.MkdirTemp("", "hmaptest") 72 | if err != nil { 73 | t.Error(err) 74 | } 75 | return tmpdir, err 76 | } 77 | 78 | func utiltestRemoveDb(t *testing.T, db DB, pbpath string) { 79 | db.Close() 80 | os.RemoveAll(pbpath) 81 | } 82 | -------------------------------------------------------------------------------- /cmd/filekv/filekv.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "strings" 8 | 9 | "github.com/projectdiscovery/hmap/filekv" 10 | fileutil "github.com/projectdiscovery/utils/file" 11 | ) 12 | 13 | func main() { 14 | // create 4 lists 15 | // two on disk 16 | list1, list2, list3 := "list1.txt", "list2.txt", "list3.txt" 17 | fList1, _ := os.Create(list1) 18 | for i := 0; i < 100000; i++ { 19 | _, _ = fList1.WriteString(fmt.Sprintf("%d\n", i)) 20 | } 21 | fList1.Close() 22 | 23 | fList2, _ := os.Create(list2) 24 | // 1000 items overlaps 25 | for i := 90000; i < 200000; i++ { 26 | _, _ = fList2.WriteString(fmt.Sprintf("%d\n", i)) 27 | } 28 | fList2.Close() 29 | 30 | // third list is still on disk but will be used as io.reader 31 | fList3, _ := os.Create(list3) 32 | // 1000 items overlaps 33 | for i := 190000; i < 300000; i++ { 34 | _, _ = fList3.WriteString(fmt.Sprintf("%d\n", i)) 35 | } 36 | fList3.Close() 37 | 38 | // 4th list will be a list of new-line separated numbers 39 | var list4 strings.Builder 40 | for i := 290000; i < 400000; i++ { 41 | list4.WriteString(fmt.Sprintf("%d\n", i)) 42 | } 43 | 44 | opts := filekv.DefaultOptions 45 | opts.Cleanup = true 46 | opts.Compress = true 47 | opts.Dedupe = filekv.MemoryLRU 48 | opts.SkipEmpty = true 49 | opts.Path = "fkv" 50 | fkv, err := filekv.Open(opts) 51 | if err != nil { 52 | log.Fatal(err) 53 | } 54 | defer fkv.Close() 55 | 56 | // opens various reader types 57 | flist3, _ := os.Open(list3) 58 | 59 | // add the various lists 60 | if _, err := fkv.Merge(list1, list2, flist3, strings.Split(list4.String(), filekv.NewLine)); err != nil { 61 | log.Fatal(err) 62 | } 63 | 64 | if err := fkv.Process(); err != nil { 65 | log.Fatal(err) 66 | } 67 | 68 | // scan the content 69 | readCount := 0 70 | err = fkv.Scan(func(b1, b2 []byte) error { 71 | readCount++ 72 | return nil 73 | }) 74 | if err != nil { 75 | log.Fatal(err) 76 | } 77 | fileutil.RemoveAll(list1, list2, list3) 78 | if readCount != 400000 { 79 | log.Fatalf("Expected 400000 but got %d\n", readCount) 80 | } else { 81 | log.Printf("Expected 400000 got %d\n", readCount) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /filekv/merge.go: -------------------------------------------------------------------------------- 1 | package filekv 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "io" 7 | "os" 8 | ) 9 | 10 | func (f *FileDB) Merge(items ...interface{}) (uint, error) { 11 | var count uint 12 | for _, item := range items { 13 | switch itemData := item.(type) { 14 | case [][]byte: 15 | for _, data := range itemData { 16 | if _, err := f.tmpDbWriter.Write(data); err != nil { 17 | return 0, err 18 | } 19 | if _, err := f.tmpDbWriter.Write([]byte(NewLine)); err != nil { 20 | return 0, err 21 | } 22 | count++ 23 | f.stats.NumberOfAddedItems++ 24 | } 25 | case []string: 26 | for _, data := range itemData { 27 | _, err := f.tmpDbWriter.Write([]byte(data + NewLine)) 28 | if err != nil { 29 | return 0, err 30 | } 31 | count++ 32 | f.stats.NumberOfAddedItems++ 33 | } 34 | case io.Reader: 35 | c, err := f.MergeReader(itemData) 36 | if err != nil { 37 | return 0, err 38 | } 39 | count += c 40 | case string: 41 | c, err := f.MergeFile(itemData) 42 | if err != nil { 43 | return 0, err 44 | } 45 | count += c 46 | } 47 | } 48 | return count, nil 49 | } 50 | 51 | func (f *FileDB) shouldSkip(k, v []byte) bool { 52 | if f.options.SkipEmpty && len(k) == 0 { 53 | return true 54 | } 55 | 56 | if f.options.FilterCallback != nil { 57 | return f.options.FilterCallback(k, v) 58 | } 59 | 60 | return false 61 | } 62 | 63 | func (f *FileDB) MergeFile(filename string) (uint, error) { 64 | newF, err := os.Open(filename) 65 | if err != nil { 66 | return 0, err 67 | } 68 | defer newF.Close() 69 | 70 | return f.MergeReader(newF) 71 | } 72 | 73 | func (f *FileDB) MergeReader(reader io.Reader) (uint, error) { 74 | var count uint 75 | sc := bufio.NewScanner(reader) 76 | buf := make([]byte, BufferSize) 77 | sc.Buffer(buf, BufferSize) 78 | var itemToWrite bytes.Buffer 79 | for sc.Scan() { 80 | itemToWrite.Write(sc.Bytes()) 81 | itemToWrite.WriteString(NewLine) 82 | if _, err := f.tmpDbWriter.Write(itemToWrite.Bytes()); err != nil { 83 | return 0, err 84 | } 85 | itemToWrite.Reset() 86 | count++ 87 | f.stats.NumberOfAddedItems++ 88 | } 89 | return count, nil 90 | } 91 | -------------------------------------------------------------------------------- /store/disk/kv_test.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "runtime" 7 | "testing" 8 | 9 | "github.com/projectdiscovery/hmap/filekv" 10 | fileutil "github.com/projectdiscovery/utils/file" 11 | stringsutil "github.com/projectdiscovery/utils/strings" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestKV(t *testing.T) { 16 | testOperations := TestOperations{ 17 | Set: true, 18 | Get: true, 19 | Scan: true, 20 | Delete: true, 21 | } 22 | var db DB 23 | // bbolt 24 | dbpath, _ := utiltestGetPath(t) 25 | dbpath = filepath.Join(dbpath, "boltdb") 26 | db, err := OpenBoltDBB(dbpath) 27 | require.Nil(t, err) 28 | db.(*BBoltDB).BucketName = "test" 29 | utiltestOperations(t, db, 100, testOperations) 30 | utiltestRemoveDb(t, db, dbpath) 31 | 32 | // pogreb 33 | dbpath, _ = utiltestGetPath(t) 34 | db, err = OpenPogrebDB(dbpath) 35 | if runtime.GOOS == "windows" && stringsutil.EqualFoldAny(runtime.GOARCH, "arm", "arm64") { 36 | require.ErrorIs(t, ErrNotSupported, err) 37 | } else { 38 | require.Nil(t, err) 39 | } 40 | utiltestOperations(t, db, 100, testOperations) 41 | utiltestRemoveDb(t, db, dbpath) 42 | 43 | // leveldb 44 | dbpath, _ = utiltestGetPath(t) 45 | db, err = OpenLevelDB(dbpath) 46 | require.Nil(t, err) 47 | utiltestOperations(t, db, 100, testOperations) 48 | utiltestRemoveDb(t, db, dbpath) 49 | 50 | // buntdb 51 | dbpath, _ = utiltestGetPath(t) 52 | dbpath = filepath.Join(dbpath, "buntdb") 53 | db, err = OpenBuntDB(dbpath) 54 | require.Nil(t, err) 55 | utiltestOperations(t, db, 100, testOperations) 56 | utiltestRemoveDb(t, db, dbpath) 57 | } 58 | 59 | func TestFileKV(t *testing.T) { 60 | dbpath, _ := fileutil.GetTempFileName() 61 | os.RemoveAll(dbpath) 62 | opts := filekv.DefaultOptions 63 | opts.Cleanup = true 64 | opts.Compress = true 65 | opts.Path = dbpath 66 | 67 | fkv, err := filekv.Open(opts) 68 | require.Nil(t, err) 69 | _, _ = fkv.Merge([]string{"a", "b"}, []string{"b", "c"}) 70 | _ = fkv.Process() 71 | count := 0 72 | _ = fkv.Scan(func(b1, b2 []byte) error { 73 | count++ 74 | return nil 75 | }) 76 | require.Equalf(t, 3, count, "wanted 3 but got %d", count) 77 | } 78 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hmap 2 | 3 | Hybrid memory/disk map that helps you to manage key value storage for input deduplication. 4 | 5 | --- 6 | 7 | Available functions: 8 | 9 | |Name|Declaration/Params/Return| 10 | |-|-| 11 | |New|func New(options Options) (*HybridMap, error){}| 12 | |Close|func (hm *HybridMap) Close() error{}| 13 | |Set|func (hm *HybridMap) Set(k string, v []byte) error{}| 14 | |Get|func (hm *HybridMap) Get(k string) ([]byte, bool){}| 15 | |Del|func (hm *HybridMap) Del(key string) error{}| 16 | |Scan|func (hm *HybridMap) Scan(f func([]byte, []byte) error){}| 17 | |Size|func (hm *HybridMap) Size() int64{}| 18 | |TuneMemory|func (hm *HybridMap) TuneMemory(){}| 19 | 20 | Available options: 21 | 22 | ```go 23 | const ( 24 | Memory MapType = iota 25 | Disk 26 | Hybrid 27 | ) 28 | 29 | type DBType int 30 | 31 | const ( 32 | LevelDB DBType = iota 33 | PogrebDB 34 | BBoltDB 35 | PebbleDB 36 | BuntDB 37 | ) 38 | ``` 39 | 40 | |Name|Props| 41 | |-|-| 42 | |`DefaultOptions`|- Type: Memory
- MemoryExpirationTime: time.Duration(5) * time.Minute
- JanitorTime: time.Duration(1) * time.Minute| 43 | |`DefaultMemoryOptions`|- Type: Memory| 44 | |`DefaultDiskOptions`|- Type: Disk
- DBType: LevelDB
- Cleanup: true
- RemoveOlderThan: 24* time.Hour *2| 45 | |`DefaultDiskOptions`|- Type: Hybrid
- DBType: PogrebDB
- MemoryExpirationTime: time.Duration(5) * time.Minute
- JanitorTime: time.Duration(1) * time.Minute| 46 | 47 | Custom options: 48 | ```go 49 | type Options struct { 50 | MemoryExpirationTime time.Duration 51 | DiskExpirationTime time.Duration 52 | JanitorTime time.Duration 53 | Type MapType 54 | DBType DBType 55 | MemoryGuardForceDisk bool 56 | MemoryGuard bool 57 | MaxMemorySize int 58 | MemoryGuardTime time.Duration 59 | Path string 60 | Cleanup bool 61 | Name string 62 | RemoveOlderThan time.Duration 63 | } 64 | ``` 65 | 66 | # Simple usage example 67 | 68 | ```go 69 | func main() { 70 | var wg sync.WaitGroup 71 | wg.Add(1) 72 | go normal(&wg) 73 | wg.Wait() 74 | } 75 | 76 | func normal(wg *sync.WaitGroup) { 77 | defer wg.Done() 78 | hm, err := hybrid.New(hybrid.DefaultOptions) 79 | if err != nil { 80 | log.Fatal(err) 81 | } 82 | defer hm.Close() 83 | err2 := hm.Set("a", []byte("b")) 84 | if err2 != nil { 85 | log.Fatal(err2) 86 | } 87 | v, ok := hm.Get("a") 88 | if ok { 89 | log.Println(v) 90 | } 91 | } 92 | ``` 93 | 94 | # License 95 | hmap is distributed under MIT License -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/projectdiscovery/hmap 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/akrylysov/pogreb v0.10.1 7 | github.com/bits-and-blooms/bloom/v3 v3.3.1 8 | github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b 9 | github.com/hashicorp/golang-lru/v2 v2.0.1 10 | github.com/pkg/errors v0.9.1 11 | github.com/projectdiscovery/utils v0.0.7 12 | github.com/rs/xid v1.4.0 13 | github.com/stretchr/testify v1.8.1 14 | github.com/syndtr/goleveldb v1.0.0 15 | github.com/tidwall/buntdb v1.2.10 16 | go.etcd.io/bbolt v1.3.7 17 | ) 18 | 19 | require ( 20 | github.com/DataDog/zstd v1.5.2 // indirect 21 | github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect 22 | github.com/aymerick/douceur v0.2.0 // indirect 23 | github.com/beorn7/perks v1.0.1 // indirect 24 | github.com/bits-and-blooms/bitset v1.3.1 // indirect 25 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 26 | github.com/cockroachdb/errors v1.9.0 // indirect 27 | github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect 28 | github.com/cockroachdb/redact v1.1.3 // indirect 29 | github.com/davecgh/go-spew v1.1.1 // indirect 30 | github.com/getsentry/sentry-go v0.16.0 // indirect 31 | github.com/gogo/protobuf v1.3.2 // indirect 32 | github.com/golang/protobuf v1.5.2 // indirect 33 | github.com/golang/snappy v0.0.4 // indirect 34 | github.com/gorilla/css v1.0.0 // indirect 35 | github.com/klauspost/compress v1.15.13 // indirect 36 | github.com/kr/pretty v0.3.1 // indirect 37 | github.com/kr/text v0.2.0 // indirect 38 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 39 | github.com/microcosm-cc/bluemonday v1.0.21 // indirect 40 | github.com/onsi/ginkgo v1.16.4 // indirect 41 | github.com/onsi/gomega v1.16.0 // indirect 42 | github.com/pmezard/go-difflib v1.0.0 // indirect 43 | github.com/prometheus/client_golang v1.14.0 // indirect 44 | github.com/prometheus/client_model v0.3.0 // indirect 45 | github.com/prometheus/common v0.39.0 // indirect 46 | github.com/prometheus/procfs v0.9.0 // indirect 47 | github.com/rogpeppe/go-internal v1.9.0 // indirect 48 | github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect 49 | github.com/tidwall/btree v1.4.3 // indirect 50 | github.com/tidwall/gjson v1.14.3 // indirect 51 | github.com/tidwall/grect v0.1.4 // indirect 52 | github.com/tidwall/match v1.1.1 // indirect 53 | github.com/tidwall/pretty v1.2.0 // indirect 54 | github.com/tidwall/rtred v0.1.2 // indirect 55 | github.com/tidwall/tinyqueue v0.1.1 // indirect 56 | golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 // indirect 57 | golang.org/x/net v0.4.0 // indirect 58 | golang.org/x/sys v0.4.0 // indirect 59 | golang.org/x/text v0.5.0 // indirect 60 | google.golang.org/protobuf v1.28.1 // indirect 61 | gopkg.in/yaml.v3 v3.0.1 // indirect 62 | ) 63 | -------------------------------------------------------------------------------- /store/disk/buntdb.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | "sync" 7 | "time" 8 | 9 | "github.com/tidwall/buntdb" 10 | ) 11 | 12 | // BuntDB - represents a BuntDB implementation 13 | type BuntDB struct { 14 | db *buntdb.DB 15 | sync.RWMutex 16 | } 17 | 18 | // OpenBuntDB - Opens the specified path 19 | func OpenBuntDB(path string) (*BuntDB, error) { 20 | db, err := buntdb.Open(path) 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | bdb := new(BuntDB) 26 | bdb.db = db 27 | 28 | return bdb, nil 29 | } 30 | 31 | // Size - Not implemented 32 | func (bdb *BuntDB) Size() int64 { 33 | return 0 34 | } 35 | 36 | // Close 37 | func (bdb *BuntDB) Close() { 38 | bdb.db.Close() 39 | } 40 | 41 | // GC - runs the garbage collector 42 | func (bdb *BuntDB) GC() error { 43 | return bdb.db.Shrink() 44 | } 45 | 46 | // Incr - increment the key by the specified value 47 | func (bdb *BuntDB) Incr(k string, by int64) (int64, error) { 48 | bdb.Lock() 49 | defer bdb.Unlock() 50 | 51 | var valP int64 52 | err := bdb.db.Update(func(tx *buntdb.Tx) error { 53 | val, err := tx.Get(k) 54 | if err != nil { 55 | return err 56 | } 57 | valP, _ = strconv.ParseInt(val, 10, 64) 58 | valP += by 59 | _, _, err = tx.Set(k, strconv.FormatInt(valP, 10), nil) 60 | return err 61 | }) 62 | 63 | return valP, err 64 | } 65 | 66 | // Set - sets a key with the specified value and optional ttl 67 | func (bdb *BuntDB) Set(k string, v []byte, ttl time.Duration) error { 68 | return bdb.db.Update(func(tx *buntdb.Tx) error { 69 | opts := new(buntdb.SetOptions) 70 | opts.Expires = ttl > 0 71 | opts.TTL = ttl 72 | _, _, err := tx.Set(k, string(v), opts) 73 | return err 74 | }) 75 | } 76 | 77 | // MSet - sets multiple key-value pairs 78 | func (bdb *BuntDB) MSet(data map[string][]byte) error { 79 | return bdb.db.Update(func(tx *buntdb.Tx) error { 80 | for k, v := range data { 81 | if _, _, err := tx.Set(k, string(v), nil); err != nil { 82 | return err 83 | } 84 | } 85 | return nil 86 | }) 87 | } 88 | 89 | // Get - fetches the value of the specified k 90 | func (bdb *BuntDB) Get(k string) ([]byte, error) { 91 | var data []byte 92 | err := bdb.db.View(func(tx *buntdb.Tx) error { 93 | val, err := tx.Get(k) 94 | if err != nil { 95 | return err 96 | } 97 | data = []byte(val) 98 | return nil 99 | }) 100 | return data, err 101 | } 102 | 103 | // MGet - fetch multiple values of the specified keys 104 | func (bdb *BuntDB) MGet(keys []string) [][]byte { 105 | var data [][]byte 106 | _ = bdb.db.View(func(tx *buntdb.Tx) error { 107 | for _, k := range keys { 108 | val, err := tx.Get(k) 109 | if err != nil { 110 | data = append(data, []byte{}) 111 | } else { 112 | data = append(data, []byte(val)) 113 | } 114 | } 115 | return nil 116 | }) 117 | return data 118 | } 119 | 120 | // TTL - returns the time to live of the specified key's value 121 | func (bdb *BuntDB) TTL(key string) int64 { 122 | var ttl int64 123 | _ = bdb.db.View(func(tx *buntdb.Tx) error { 124 | d, err := tx.TTL(key) 125 | if err != nil { 126 | return err 127 | } 128 | ttl = int64(d) 129 | return nil 130 | }) 131 | return ttl 132 | } 133 | 134 | // MDel - removes key(s) from the store 135 | func (bdb *BuntDB) MDel(keys []string) error { 136 | return bdb.db.Update(func(tx *buntdb.Tx) error { 137 | for _, k := range keys { 138 | if _, err := tx.Delete(k); err != nil { 139 | return err 140 | } 141 | } 142 | return nil 143 | }) 144 | } 145 | 146 | // Del - removes key from the store 147 | func (bdb *BuntDB) Del(key string) error { 148 | return bdb.db.Update(func(tx *buntdb.Tx) error { 149 | _, err := tx.Delete(key) 150 | if err != nil { 151 | return err 152 | } 153 | return nil 154 | }) 155 | } 156 | 157 | // Scan - iterate over the whole store using the handler function 158 | func (bdb *BuntDB) Scan(opt ScannerOptions) error { 159 | valid := func(k, v string) bool { 160 | // Do not include offset item, skip this 161 | if !opt.IncludeOffset && len(opt.Offset) > 0 && k == opt.Offset { 162 | return true 163 | } 164 | 165 | // Do not has prefix, iterate out of bound, exit 166 | if len(opt.Prefix) > 0 && !strings.HasPrefix(k, opt.Prefix) { 167 | return false 168 | } 169 | 170 | if opt.Handler([]byte(k), []byte(v)) != nil { 171 | return false 172 | } 173 | return true 174 | } 175 | return bdb.db.View(func(tx *buntdb.Tx) error { 176 | // Has offset 177 | if len(opt.Offset) > 0 { 178 | return tx.AscendGreaterOrEqual("", opt.Offset, valid) 179 | } 180 | 181 | // Only prefix 182 | if len(opt.Prefix) > 0 && len(opt.Offset) == 0 { 183 | return tx.AscendKeys(opt.Prefix+"*", valid) 184 | } 185 | 186 | return tx.Ascend("", valid) 187 | }) 188 | } 189 | -------------------------------------------------------------------------------- /cmd/example/example.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "sync" 7 | "time" 8 | 9 | "github.com/pkg/errors" 10 | "github.com/projectdiscovery/hmap/store/hybrid" 11 | ) 12 | 13 | func main() { 14 | var wg sync.WaitGroup 15 | wg.Add(1) 16 | go normal(&wg) 17 | wg.Add(1) 18 | go memoryExpire(&wg) 19 | wg.Add(1) 20 | go disk(&wg) 21 | wg.Add(1) 22 | go hybridz(&wg) 23 | wg.Wait() 24 | wg.Add(1) 25 | _ = allDisks(&wg) 26 | wg.Wait() 27 | } 28 | 29 | func normal(wg *sync.WaitGroup) { 30 | defer wg.Done() 31 | hm, err := hybrid.New(hybrid.DefaultOptions) 32 | if err != nil { 33 | log.Fatal(err) 34 | } 35 | defer hm.Close() 36 | err2 := hm.Set("a", []byte("b")) 37 | if err2 != nil { 38 | log.Fatal(err2) 39 | } 40 | v, ok := hm.Get("a") 41 | if ok { 42 | log.Println(v) 43 | } 44 | } 45 | 46 | func memoryExpire(wg *sync.WaitGroup) { 47 | defer wg.Done() 48 | hm, err := hybrid.New(hybrid.Options{ 49 | Type: hybrid.Memory, 50 | MemoryExpirationTime: time.Duration(10) * time.Second, 51 | JanitorTime: time.Duration(5) * time.Second, 52 | }) 53 | if err != nil { 54 | log.Fatal(err) 55 | } 56 | defer hm.Close() 57 | err2 := hm.Set("a", []byte("b")) 58 | if err2 != nil { 59 | log.Fatal(err2) 60 | } 61 | time.Sleep(time.Duration(15) * time.Second) 62 | v, ok := hm.Get("a") 63 | if ok && len(v) == 0 { 64 | log.Printf("error: item should be evicted") 65 | return 66 | } 67 | log.Printf("item evicted") 68 | } 69 | 70 | func disk(wg *sync.WaitGroup) { 71 | defer wg.Done() 72 | hm, err := hybrid.New(hybrid.DefaultDiskOptions) 73 | if err != nil { 74 | log.Fatal(err) 75 | } 76 | defer hm.Close() 77 | err2 := hm.Set("a", []byte("b")) 78 | if err2 != nil { 79 | log.Fatal(err2) 80 | } 81 | v, ok := hm.Get("a") 82 | if ok { 83 | log.Println(string(v)) 84 | return 85 | } 86 | log.Printf("error: not found") 87 | } 88 | 89 | func hybridz(wg *sync.WaitGroup) { 90 | defer wg.Done() 91 | hm, err := hybrid.New(hybrid.Options{ 92 | Type: hybrid.Hybrid, 93 | MemoryExpirationTime: time.Duration(60) * time.Second, 94 | MemoryGuard: true, 95 | MemoryGuardTime: time.Duration(5) * time.Second, 96 | JanitorTime: time.Duration(5) * time.Second, 97 | MaxMemorySize: 1024, 98 | }) 99 | if err != nil { 100 | log.Fatal(err) 101 | } 102 | defer hm.Close() 103 | err2 := hm.Set("a", []byte("b")) 104 | if err2 != nil { 105 | log.Fatal(err2) 106 | } 107 | time.Sleep(time.Duration(15) * time.Second) 108 | // this should happen from disk 109 | v, ok := hm.Get("a") 110 | if ok { 111 | log.Println("Read1 (disk)", v) 112 | } else { 113 | log.Println("Read1 (disk) Not found") 114 | } 115 | 116 | log.Println("Writing 10k") 117 | for i := 0; i < 10000; i++ { 118 | v := fmt.Sprintf("%d", i) 119 | _ = hm.Set(v, []byte(v)) 120 | } 121 | log.Println("Finished writing 10k") 122 | 123 | time.Sleep(time.Duration(15) * time.Second) 124 | // this should happen from memory again 125 | v3, ok3 := hm.Get("a") 126 | if ok3 { 127 | log.Println("Read2 (memory)", v3) 128 | } else { 129 | log.Println("Read2 (memory) Not found") 130 | } 131 | } 132 | 133 | func allDisks(wg *sync.WaitGroup) error { 134 | defer wg.Done() 135 | 136 | total := 10000 137 | 138 | opts := hybrid.DefaultDiskOptions 139 | // leveldb 140 | opts.DBType = hybrid.LevelDB 141 | _, err := testhybrid("leveldb", opts, total) 142 | if err != nil { 143 | return err 144 | } 145 | 146 | // pogreb 147 | opts.DBType = hybrid.PogrebDB 148 | _, err = testhybrid("pogreb", opts, total) 149 | if err != nil { 150 | return err 151 | } 152 | 153 | // bbolt 154 | opts.DBType = hybrid.BBoltDB 155 | opts.Name = "test" 156 | _, err = testhybrid("bbolt", opts, total) 157 | if err != nil { 158 | return err 159 | } 160 | 161 | // buntdb 162 | opts.DBType = hybrid.BuntDB 163 | _, err = testhybrid("buntdb", opts, total) 164 | if err != nil { 165 | return err 166 | } 167 | 168 | return nil 169 | } 170 | 171 | func testhybrid(name string, opts hybrid.Options, total int) (duration time.Duration, err error) { 172 | start := time.Now() 173 | 174 | log.Println("starting:", name) 175 | 176 | hm, err := hybrid.New(opts) 177 | if err != nil { 178 | log.Fatal(err) 179 | } 180 | 181 | // write 182 | written := 0 183 | for i := 0; i < total; i++ { 184 | if err = hm.Set(fmt.Sprint(i), []byte("test")); err != nil { 185 | log.Fatal(err) 186 | duration = time.Since(start) 187 | return 188 | } 189 | written++ 190 | } 191 | 192 | // scan 193 | read := 0 194 | hm.Scan(func(k, v []byte) error { 195 | gotValue := string(v) 196 | if "test" != gotValue { 197 | return errors.New("unexpected item") 198 | } 199 | read++ 200 | return nil 201 | }) 202 | 203 | err = hm.Close() 204 | duration = time.Since(start) 205 | log.Printf("written: %d,read %d, took:%s\n", written, read, duration) 206 | return 207 | } 208 | -------------------------------------------------------------------------------- /store/disk/bboltdb.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "bytes" 5 | "strconv" 6 | "sync" 7 | "time" 8 | 9 | "github.com/pkg/errors" 10 | 11 | bolt "go.etcd.io/bbolt" 12 | ) 13 | 14 | // BBoltDB - represents a bbolt db implementation 15 | type BBoltDB struct { 16 | db *bolt.DB 17 | sync.RWMutex 18 | BucketName string 19 | } 20 | 21 | // OpenBoltDB - Opens the specified path 22 | func OpenBoltDBB(path string) (*BBoltDB, error) { 23 | db, err := bolt.Open(path, 0600, nil) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | bbdb := new(BBoltDB) 29 | bbdb.db = db 30 | 31 | return bbdb, nil 32 | } 33 | 34 | // Size - returns the size of the database in bytes 35 | func (b *BBoltDB) Size() int64 { 36 | // not implemented 37 | return 0 38 | } 39 | 40 | // Close ... 41 | func (b *BBoltDB) Close() { 42 | b.db.Close() 43 | } 44 | 45 | // GC - runs the garbage collector 46 | func (b *BBoltDB) GC() error { 47 | return ErrNotImplemented 48 | } 49 | 50 | // Incr - increment the key by the specified value 51 | func (b *BBoltDB) Incr(k string, by int64) (int64, error) { 52 | return 0, ErrNotImplemented 53 | } 54 | 55 | func (b *BBoltDB) set(k, v []byte, ttl time.Duration) error { 56 | return b.db.Update(func(tx *bolt.Tx) error { 57 | var expires int64 58 | if ttl > 0 { 59 | expires = time.Now().Add(ttl).Unix() 60 | } 61 | b, err := tx.CreateBucketIfNotExists([]byte(b.BucketName)) 62 | if err != nil { 63 | return err 64 | } 65 | expiresBytes := append(intToByteSlice(expires), expSeparator[:]...) 66 | v = append(expiresBytes, v...) 67 | return b.Put(k, v) 68 | }) 69 | } 70 | 71 | // Set - sets a key with the specified value and optional ttl 72 | func (b *BBoltDB) Set(k string, v []byte, ttl time.Duration) error { 73 | return b.set([]byte(k), v, ttl) 74 | } 75 | 76 | // MSet - sets multiple key-value pairs 77 | func (b *BBoltDB) MSet(data map[string][]byte) error { 78 | return ErrNotImplemented 79 | } 80 | 81 | func (b *BBoltDB) get(k string) ([]byte, error) { 82 | var data []byte 83 | delete := false 84 | 85 | return data, b.db.Update(func(tx *bolt.Tx) error { 86 | b, err := tx.CreateBucketIfNotExists([]byte(b.BucketName)) 87 | if err != nil { 88 | return err 89 | } 90 | data = b.Get([]byte(k)) 91 | if data == nil { 92 | return ErrNoData 93 | } 94 | parts := bytes.SplitN(data, []byte(expSeparator), 2) 95 | expires, actual := parts[0], parts[1] 96 | if exp, _ := strconv.Atoi(string(expires)); exp > 0 && int(time.Now().Unix()) >= exp { 97 | delete = true 98 | } 99 | data = actual 100 | 101 | if delete { 102 | return b.Delete([]byte(k)) 103 | } 104 | 105 | return nil 106 | }) 107 | } 108 | 109 | // Get - fetches the value of the specified k 110 | func (b *BBoltDB) Get(k string) ([]byte, error) { 111 | return b.get(k) 112 | } 113 | 114 | // MGet - fetch multiple values of the specified keys 115 | func (b *BBoltDB) MGet(keys []string) [][]byte { 116 | var data [][]byte 117 | for _, key := range keys { 118 | val, err := b.get(key) 119 | if err != nil { 120 | data = append(data, []byte{}) 121 | continue 122 | } 123 | data = append(data, val) 124 | } 125 | return data 126 | } 127 | 128 | // TTL - returns the time to live of the specified key's value 129 | func (b *BBoltDB) TTL(key string) int64 { 130 | item, err := b.get(key) 131 | if err != nil { 132 | return -2 133 | } 134 | 135 | parts := bytes.SplitN(item, []byte(expSeparator), 2) 136 | exp, _ := strconv.Atoi(string(parts[0])) 137 | if exp == 0 { 138 | return -1 139 | } 140 | 141 | now := time.Now().Unix() 142 | if now >= int64(exp) { 143 | return -2 144 | } 145 | 146 | return int64(exp) - now 147 | } 148 | 149 | // MDel - removes key(s) from the store 150 | func (b *BBoltDB) MDel(keys []string) error { 151 | return ErrNotImplemented 152 | } 153 | 154 | // Del - removes key from the store 155 | func (b *BBoltDB) Del(key string) error { 156 | return b.db.Update(func(tx *bolt.Tx) error { 157 | b, err := tx.CreateBucketIfNotExists([]byte(b.BucketName)) 158 | if err != nil { 159 | return err 160 | } 161 | return b.Delete([]byte(key)) 162 | }) 163 | } 164 | 165 | // Scan - iterate over the whole store using the handler function 166 | func (b *BBoltDB) Scan(scannerOpt ScannerOptions) error { 167 | valid := func(k []byte) bool { 168 | if k == nil { 169 | return false 170 | } 171 | 172 | if scannerOpt.Prefix != "" && !bytes.HasPrefix(k, []byte(scannerOpt.Prefix)) { 173 | return false 174 | } 175 | 176 | return true 177 | } 178 | return b.db.View(func(tx *bolt.Tx) error { 179 | b := tx.Bucket([]byte(b.BucketName)) 180 | if b == nil { 181 | return errors.New("bucket not found") 182 | } 183 | c := b.Cursor() 184 | for key, val := c.First(); key != nil; key, val = c.Next() { 185 | parts := bytes.SplitN(val, []byte(expSeparator), 2) 186 | data := val 187 | if len(parts) == 2 { 188 | data = parts[1] 189 | } 190 | if !valid(key) || scannerOpt.Handler(key, data) != nil { 191 | break 192 | } 193 | } 194 | return nil 195 | }) 196 | } 197 | -------------------------------------------------------------------------------- /store/disk/pebble_all.go: -------------------------------------------------------------------------------- 1 | //go:build !(386 || arm) 2 | 3 | package disk 4 | 5 | import ( 6 | "bytes" 7 | "strconv" 8 | "sync" 9 | "time" 10 | 11 | "github.com/cockroachdb/pebble" 12 | ) 13 | 14 | func init() { 15 | OpenPebbleDB = openPebbleDB 16 | } 17 | 18 | // PebbleDB - represents a pebble db implementation 19 | type PebbleDB struct { 20 | db *pebble.DB 21 | sync.RWMutex 22 | } 23 | 24 | // openPebbleDB - Opens the specified path 25 | func openPebbleDB(path string) (DB, error) { 26 | db, err := pebble.Open(path, &pebble.Options{}) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | pdb := new(PebbleDB) 32 | pdb.db = db 33 | 34 | return pdb, nil 35 | } 36 | 37 | // Size - returns the size of the database in bytes 38 | func (pdb *PebbleDB) Size() int64 { 39 | metrics := pdb.db.Metrics() 40 | if metrics == nil { 41 | return 0 42 | } 43 | return metrics.Total().Size 44 | } 45 | 46 | // Close ... 47 | func (pdb *PebbleDB) Close() { 48 | pdb.db.Close() 49 | } 50 | 51 | // GC - runs the garbage collector 52 | func (pdb *PebbleDB) GC() error { 53 | // find first and last key 54 | iter := pdb.db.NewIter(&pebble.IterOptions{}) 55 | first := iter.Key() 56 | var last []byte 57 | for iter.Next() { 58 | if iter.Last() { 59 | last = iter.Key() 60 | } 61 | } 62 | return pdb.db.Compact(first, last, false) 63 | } 64 | 65 | // Incr - increment the key by the specified value 66 | func (pdb *PebbleDB) Incr(k string, by int64) (int64, error) { 67 | pdb.Lock() 68 | defer pdb.Unlock() 69 | 70 | val, err := pdb.get(k) 71 | if err != nil { 72 | val = []byte{} 73 | } 74 | 75 | valFloat, _ := strconv.ParseInt(string(val), 10, 64) 76 | valFloat += by 77 | 78 | err = pdb.set([]byte(k), intToByteSlice(valFloat), -1) 79 | if err != nil { 80 | return 0, err 81 | } 82 | 83 | return valFloat, nil 84 | } 85 | 86 | func (pdb *PebbleDB) set(k, v []byte, ttl time.Duration) error { 87 | var expires int64 88 | if ttl > 0 { 89 | expires = time.Now().Add(ttl).Unix() 90 | } 91 | expiresBytes := append(intToByteSlice(expires), expSeparator[:]...) 92 | v = append(expiresBytes, v...) 93 | return pdb.db.Set(k, v, pebble.Sync) 94 | } 95 | 96 | // Set - sets a key with the specified value and optional ttl 97 | func (pdb *PebbleDB) Set(k string, v []byte, ttl time.Duration) error { 98 | return pdb.set([]byte(k), v, ttl) 99 | } 100 | 101 | // MSet - sets multiple key-value pairs 102 | func (pdb *PebbleDB) MSet(data map[string][]byte) error { 103 | return nil 104 | } 105 | 106 | func (pdb *PebbleDB) get(k string) ([]byte, error) { 107 | var data []byte 108 | var err error 109 | 110 | delete := false 111 | 112 | s, closer, err := pdb.db.Get([]byte(k)) 113 | if err != nil { 114 | return []byte{}, err 115 | } 116 | defer closer.Close() 117 | 118 | // make a copy of the byte slice as we need to return it safely 119 | item := append(s[:0:0], s...) 120 | 121 | parts := bytes.SplitN(item, []byte(expSeparator), 2) 122 | expires, actual := parts[0], parts[1] 123 | 124 | if exp, _ := strconv.Atoi(string(expires)); exp > 0 && int(time.Now().Unix()) >= exp { 125 | delete = true 126 | } else { 127 | data = actual 128 | } 129 | 130 | if delete { 131 | err := pdb.db.Delete([]byte(k), pebble.Sync) 132 | if err != nil { 133 | return data, err 134 | } 135 | return data, ErrNotFound 136 | } 137 | 138 | return data, nil 139 | } 140 | 141 | // Get - fetches the value of the specified k 142 | func (pdb *PebbleDB) Get(k string) ([]byte, error) { 143 | return pdb.get(k) 144 | } 145 | 146 | // MGet - fetch multiple values of the specified keys 147 | func (pdb *PebbleDB) MGet(keys []string) [][]byte { 148 | var data [][]byte 149 | for _, key := range keys { 150 | val, err := pdb.get(key) 151 | if err != nil { 152 | data = append(data, []byte{}) 153 | continue 154 | } 155 | data = append(data, val) 156 | } 157 | return data 158 | } 159 | 160 | // TTL - returns the time to live of the specified key's value 161 | func (pdb *PebbleDB) TTL(key string) int64 { 162 | item, closer, err := pdb.db.Get([]byte(key)) 163 | if err != nil { 164 | return -2 165 | } 166 | defer closer.Close() 167 | 168 | parts := bytes.SplitN(item, []byte(expSeparator), 2) 169 | exp, _ := strconv.Atoi(string(parts[0])) 170 | if exp == 0 { 171 | return -1 172 | } 173 | 174 | now := time.Now().Unix() 175 | if now >= int64(exp) { 176 | return -2 177 | } 178 | 179 | return int64(exp) - now 180 | } 181 | 182 | // MDel - removes key(s) from the store 183 | func (pdb *PebbleDB) MDel(keys []string) error { 184 | return nil 185 | } 186 | 187 | // Del - removes key from the store 188 | func (pdb *PebbleDB) Del(key string) error { 189 | return pdb.db.Delete([]byte(key), pebble.Sync) 190 | } 191 | 192 | // Scan - iterate over the whole store using the handler function 193 | func (pdb *PebbleDB) Scan(scannerOpt ScannerOptions) error { 194 | iter := pdb.db.NewIter(nil) 195 | for iter.First(); iter.Valid(); iter.Next() { 196 | key, val := iter.Key(), iter.Value() 197 | if scannerOpt.Handler(key, val) != nil { 198 | break 199 | } 200 | } 201 | 202 | return nil 203 | } 204 | -------------------------------------------------------------------------------- /store/disk/pogreb_all.go: -------------------------------------------------------------------------------- 1 | //go:build !((arm || arm64) && windows) 2 | 3 | package disk 4 | 5 | import ( 6 | "bytes" 7 | "strconv" 8 | "sync" 9 | "time" 10 | 11 | "github.com/akrylysov/pogreb" 12 | ) 13 | 14 | func init() { 15 | OpenPogrebDB = openPogrebDB 16 | } 17 | 18 | // PogrebDB - represents a pogreb db implementation 19 | type PogrebDB struct { 20 | db *pogreb.DB 21 | sync.RWMutex 22 | } 23 | 24 | // OpenPogrebDB - Opens the specified path 25 | func openPogrebDB(path string) (DB, error) { 26 | db, err := pogreb.Open(path, nil) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | pdb := new(PogrebDB) 32 | pdb.db = db 33 | 34 | return pdb, nil 35 | } 36 | 37 | // Size - returns the size of the database in bytes 38 | func (pdb *PogrebDB) Size() int64 { 39 | size, err := pdb.db.FileSize() 40 | if err != nil { 41 | return 0 42 | } 43 | return size 44 | } 45 | 46 | // Close ... 47 | func (pdb *PogrebDB) Close() { 48 | pdb.db.Close() 49 | } 50 | 51 | // GC - runs the garbage collector 52 | func (pdb *PogrebDB) GC() error { 53 | _, err := pdb.db.Compact() 54 | return err 55 | } 56 | 57 | // Incr - increment the key by the specified value 58 | func (pdb *PogrebDB) Incr(k string, by int64) (int64, error) { 59 | pdb.Lock() 60 | defer pdb.Unlock() 61 | 62 | val, err := pdb.get(k) 63 | if err != nil { 64 | val = []byte{} 65 | } 66 | 67 | valFloat, _ := strconv.ParseInt(string(val), 10, 64) 68 | valFloat += by 69 | 70 | err = pdb.set([]byte(k), intToByteSlice(valFloat), -1) 71 | if err != nil { 72 | return 0, err 73 | } 74 | 75 | return valFloat, nil 76 | } 77 | 78 | func (pdb *PogrebDB) set(k, v []byte, ttl time.Duration) error { 79 | var expires int64 80 | if ttl > 0 { 81 | expires = time.Now().Add(ttl).Unix() 82 | } 83 | expiresBytes := append(intToByteSlice(expires), expSeparator[:]...) 84 | v = append(expiresBytes, v...) 85 | return pdb.db.Put(k, v) 86 | } 87 | 88 | // Set - sets a key with the specified value and optional ttl 89 | func (pdb *PogrebDB) Set(k string, v []byte, ttl time.Duration) error { 90 | return pdb.set([]byte(k), v, ttl) 91 | } 92 | 93 | // MSet - sets multiple key-value pairs 94 | func (pdb *PogrebDB) MSet(data map[string][]byte) error { 95 | return nil 96 | } 97 | 98 | func (pdb *PogrebDB) get(k string) ([]byte, error) { 99 | var data []byte 100 | var err error 101 | 102 | delete := false 103 | 104 | item, err := pdb.db.Get([]byte(k)) 105 | if err != nil { 106 | return []byte{}, err 107 | } 108 | 109 | if len(item) == 0 { 110 | return []byte{}, ErrNotFound 111 | } 112 | 113 | parts := bytes.SplitN(item, []byte(expSeparator), 2) 114 | expires, actual := parts[0], parts[1] 115 | 116 | if exp, _ := strconv.Atoi(string(expires)); exp > 0 && int(time.Now().Unix()) >= exp { 117 | delete = true 118 | } 119 | data = actual 120 | 121 | if delete { 122 | errDelete := pdb.db.Delete([]byte(k)) 123 | if errDelete != nil { 124 | return data, errDelete 125 | } 126 | return data, ErrNotFound 127 | } 128 | return data, nil 129 | } 130 | 131 | // Get - fetches the value of the specified k 132 | func (pdb *PogrebDB) Get(k string) ([]byte, error) { 133 | return pdb.get(k) 134 | } 135 | 136 | // MGet - fetch multiple values of the specified keys 137 | func (pdb *PogrebDB) MGet(keys []string) [][]byte { 138 | var data [][]byte 139 | for _, key := range keys { 140 | val, err := pdb.get(key) 141 | if err != nil { 142 | data = append(data, []byte{}) 143 | continue 144 | } 145 | data = append(data, val) 146 | } 147 | return data 148 | } 149 | 150 | // TTL - returns the time to live of the specified key's value 151 | func (pdb *PogrebDB) TTL(key string) int64 { 152 | item, err := pdb.db.Get([]byte(key)) 153 | if err != nil { 154 | return -2 155 | } 156 | 157 | parts := bytes.SplitN(item, []byte(expSeparator), 2) 158 | exp, _ := strconv.Atoi(string(parts[0])) 159 | if exp == 0 { 160 | return -1 161 | } 162 | 163 | now := time.Now().Unix() 164 | if now >= int64(exp) { 165 | return -2 166 | } 167 | 168 | return int64(exp) - now 169 | } 170 | 171 | // MDel - removes key(s) from the store 172 | func (pdb *PogrebDB) MDel(keys []string) error { 173 | return nil 174 | } 175 | 176 | // Del - removes key from the store 177 | func (pdb *PogrebDB) Del(key string) error { 178 | return pdb.db.Delete([]byte(key)) 179 | } 180 | 181 | // Scan - iterate over the whole store using the handler function 182 | func (pdb *PogrebDB) Scan(scannerOpt ScannerOptions) error { 183 | valid := func(k []byte) bool { 184 | if k == nil { 185 | return false 186 | } 187 | 188 | if scannerOpt.Prefix != "" && !bytes.HasPrefix(k, []byte(scannerOpt.Prefix)) { 189 | return false 190 | } 191 | 192 | return true 193 | } 194 | 195 | it := pdb.db.Items() 196 | for { 197 | key, val, err := it.Next() 198 | if err == pogreb.ErrIterationDone { 199 | break 200 | } 201 | if err != nil { 202 | return err 203 | } 204 | parts := bytes.SplitN(val, []byte(expSeparator), 2) 205 | _, data := parts[0], parts[1] 206 | if !valid(key) || scannerOpt.Handler(key, data) != nil { 207 | break 208 | } 209 | } 210 | 211 | return nil 212 | } 213 | -------------------------------------------------------------------------------- /store/cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "runtime" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | const ( 10 | NoExpiration time.Duration = -1 11 | DefaultExpiration time.Duration = 0 12 | ) 13 | 14 | type Cache interface { 15 | SetWithExpiration(string, interface{}, time.Duration) 16 | Set(string, interface{}) 17 | Get(string) (interface{}, bool) 18 | Delete(string) 19 | DeleteExpired() 20 | OnEvicted(func(string, interface{})) 21 | CloneItems() map[string]Item 22 | Scan(func([]byte, []byte) error) 23 | ItemCount() int 24 | } 25 | 26 | type CacheMemory struct { 27 | *cacheMemory 28 | } 29 | 30 | type cacheMemory struct { 31 | DefaultExpiration time.Duration 32 | Items map[string]Item 33 | mu sync.RWMutex 34 | onEvicted func(string, interface{}) 35 | janitor *janitor 36 | } 37 | 38 | func (c *cacheMemory) SetWithExpiration(k string, x interface{}, d time.Duration) { 39 | c.mu.Lock() 40 | defer c.mu.Unlock() 41 | c.set(k, x, d) 42 | } 43 | 44 | func (c *cacheMemory) set(k string, x interface{}, d time.Duration) { 45 | var e int64 46 | if d == DefaultExpiration { 47 | d = c.DefaultExpiration 48 | } 49 | if d > 0 { 50 | e = time.Now().Add(d).UnixNano() 51 | } 52 | c.Items[k] = Item{ 53 | Object: x, 54 | Expiration: e, 55 | } 56 | } 57 | 58 | func (c *cacheMemory) Set(k string, x interface{}) { 59 | c.SetWithExpiration(k, x, c.DefaultExpiration) 60 | } 61 | 62 | func (c *cacheMemory) Get(k string) (interface{}, bool) { 63 | c.mu.RLock() 64 | item, found := c.Items[k] 65 | if !found { 66 | c.mu.RUnlock() 67 | return nil, false 68 | } 69 | if item.Expiration > 0 { 70 | if time.Now().UnixNano() > item.Expiration { 71 | c.mu.RUnlock() 72 | return nil, false 73 | } 74 | } 75 | c.mu.RUnlock() 76 | c.refresh(k) 77 | return item.Object, true 78 | } 79 | 80 | func (c *cacheMemory) refresh(k string) bool { 81 | c.mu.Lock() 82 | defer c.mu.Unlock() 83 | 84 | item, found := c.Items[k] 85 | if !found { 86 | return false 87 | } 88 | item.Expiration = time.Now().Add(c.DefaultExpiration).UnixNano() 89 | return true 90 | } 91 | 92 | func (c *cacheMemory) Delete(k string) { 93 | c.mu.Lock() 94 | v, evicted := c.delete(k) 95 | c.mu.Unlock() 96 | if evicted { 97 | c.onEvicted(k, v) 98 | } 99 | } 100 | 101 | func (c *cacheMemory) delete(k string) (interface{}, bool) { 102 | if c.onEvicted != nil { 103 | if v, found := c.Items[k]; found { 104 | delete(c.Items, k) 105 | return v.Object, true 106 | } 107 | } 108 | delete(c.Items, k) 109 | return nil, false 110 | } 111 | 112 | // Delete all expired items from the cache. 113 | func (c *cacheMemory) DeleteExpired() { 114 | var evictedItems []keyAndValue 115 | now := time.Now().UnixNano() 116 | c.mu.Lock() 117 | for k, v := range c.Items { 118 | // "Inlining" of expired 119 | if v.Expiration > 0 && now > v.Expiration { 120 | ov, evicted := c.delete(k) 121 | if evicted { 122 | evictedItems = append(evictedItems, keyAndValue{k, ov}) 123 | } 124 | } 125 | } 126 | c.mu.Unlock() 127 | for _, v := range evictedItems { 128 | c.onEvicted(v.key, v.value) 129 | } 130 | } 131 | 132 | func (c *cacheMemory) OnEvicted(f func(string, interface{})) { 133 | c.mu.Lock() 134 | defer c.mu.Unlock() 135 | c.onEvicted = f 136 | } 137 | 138 | func (c *cacheMemory) Scan(f func([]byte, []byte) error) { 139 | c.mu.Lock() 140 | defer c.mu.Unlock() 141 | 142 | for k, item := range c.Items { 143 | if f([]byte(k), item.Object.([]byte)) != nil { 144 | break 145 | } 146 | } 147 | } 148 | 149 | func (c *cacheMemory) CloneItems() map[string]Item { 150 | c.mu.RLock() 151 | defer c.mu.RUnlock() 152 | m := make(map[string]Item, len(c.Items)) 153 | now := time.Now().UnixNano() 154 | for k, v := range c.Items { 155 | // "Inlining" of Expired 156 | if v.Expiration > 0 { 157 | if now > v.Expiration { 158 | continue 159 | } 160 | } 161 | m[k] = v 162 | } 163 | return m 164 | } 165 | 166 | func (c *cacheMemory) ItemCount() int { 167 | c.mu.RLock() 168 | defer c.mu.RUnlock() 169 | n := len(c.Items) 170 | 171 | return n 172 | } 173 | 174 | func (c *cacheMemory) Empty() { 175 | c.mu.Lock() 176 | defer c.mu.Unlock() 177 | 178 | c.Items = map[string]Item{} 179 | } 180 | 181 | func newCache(de time.Duration, m map[string]Item) *cacheMemory { 182 | if de == 0 { 183 | de = -1 184 | } 185 | c := &cacheMemory{ 186 | DefaultExpiration: de, 187 | Items: m, 188 | } 189 | return c 190 | } 191 | 192 | func newCacheWithJanitor(de time.Duration, ci time.Duration, m map[string]Item) *CacheMemory { 193 | c := newCache(de, m) 194 | w := &CacheMemory{ 195 | cacheMemory: c, 196 | } 197 | if ci > 0 { 198 | runJanitor(c, ci) 199 | runtime.SetFinalizer(w, func(c *CacheMemory) { 200 | stopJanitor(c.cacheMemory) 201 | }) 202 | } 203 | return w 204 | } 205 | 206 | func New(defaultExpiration, cleanupInterval time.Duration) *CacheMemory { 207 | items := make(map[string]Item) 208 | return newCacheWithJanitor(defaultExpiration, cleanupInterval, items) 209 | } 210 | 211 | func NewFrom(defaultExpiration, cleanupInterval time.Duration, items map[string]Item) *CacheMemory { 212 | return newCacheWithJanitor(defaultExpiration, cleanupInterval, items) 213 | } 214 | -------------------------------------------------------------------------------- /store/disk/leveldb.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "bytes" 5 | "strconv" 6 | "sync" 7 | "time" 8 | 9 | "github.com/syndtr/goleveldb/leveldb" 10 | "github.com/syndtr/goleveldb/leveldb/iterator" 11 | "github.com/syndtr/goleveldb/leveldb/opt" 12 | "github.com/syndtr/goleveldb/leveldb/util" 13 | ) 14 | 15 | const expSeparator = ";" 16 | 17 | // LevelDB - represents a leveldb db implementation 18 | type LevelDB struct { 19 | db *leveldb.DB 20 | sync.RWMutex 21 | } 22 | 23 | // OpenLevelDB - Opens the specified path 24 | func OpenLevelDB(path string) (*LevelDB, error) { 25 | db, err := leveldb.OpenFile(path, &opt.Options{ 26 | CompactionTableSize: 64 * Megabyte, 27 | }) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | ldb := new(LevelDB) 33 | ldb.db = db 34 | 35 | return ldb, nil 36 | } 37 | 38 | // Size - returns the size of the database in bytes 39 | func (ldb *LevelDB) Size() int64 { 40 | var stats leveldb.DBStats 41 | if nil != ldb.db.Stats(&stats) { 42 | return -1 43 | } 44 | size := int64(0) 45 | for _, v := range stats.LevelSizes { 46 | size += v 47 | } 48 | return size 49 | } 50 | 51 | // Close ... 52 | func (ldb *LevelDB) Close() { 53 | ldb.db.Close() 54 | } 55 | 56 | // GC - runs the garbage collector 57 | func (ldb *LevelDB) GC() error { 58 | return ldb.db.CompactRange(util.Range{}) 59 | } 60 | 61 | // Incr - increment the key by the specified value 62 | func (ldb *LevelDB) Incr(k string, by int64) (int64, error) { 63 | ldb.Lock() 64 | defer ldb.Unlock() 65 | 66 | val, err := ldb.get(k) 67 | if err != nil { 68 | val = []byte{} 69 | } 70 | 71 | valFloat, _ := strconv.ParseInt(string(val), 10, 64) 72 | valFloat += by 73 | 74 | err = ldb.set([]byte(k), intToByteSlice(valFloat), -1) 75 | if err != nil { 76 | return 0, err 77 | } 78 | 79 | return valFloat, nil 80 | } 81 | 82 | func intToByteSlice(v int64) []byte { 83 | return []byte(strconv.FormatInt(v, 10)) 84 | } 85 | 86 | func (ldb *LevelDB) set(k, v []byte, ttl time.Duration) error { 87 | var expires int64 88 | if ttl > 0 { 89 | expires = time.Now().Add(ttl).Unix() 90 | } 91 | expiresBytes := append(intToByteSlice(expires), expSeparator[:]...) 92 | v = append(expiresBytes, v...) 93 | return ldb.db.Put(k, v, nil) 94 | } 95 | 96 | // Set - sets a key with the specified value and optional ttl 97 | func (ldb *LevelDB) Set(k string, v []byte, ttl time.Duration) error { 98 | return ldb.set([]byte(k), v, ttl) 99 | } 100 | 101 | // MSet - sets multiple key-value pairs 102 | func (ldb *LevelDB) MSet(data map[string][]byte) error { 103 | batch := new(leveldb.Batch) 104 | for k, v := range data { 105 | v = append([]byte("0;"), v...) 106 | batch.Put([]byte(k), v) 107 | } 108 | return ldb.db.Write(batch, nil) 109 | } 110 | 111 | func (ldb *LevelDB) get(k string) ([]byte, error) { 112 | var data []byte 113 | var err error 114 | 115 | delete := false 116 | 117 | item, err := ldb.db.Get([]byte(k), nil) 118 | if err != nil { 119 | return []byte{}, err 120 | } 121 | 122 | parts := bytes.SplitN(item, []byte(expSeparator), 2) 123 | expires, actual := parts[0], parts[1] 124 | 125 | if exp, _ := strconv.Atoi(string(expires)); exp > 0 && int(time.Now().Unix()) >= exp { 126 | delete = true 127 | } else { 128 | data = actual 129 | } 130 | 131 | if delete { 132 | errDelete := ldb.db.Delete([]byte(k), nil) 133 | if errDelete != nil { 134 | return data, errDelete 135 | } 136 | return data, ErrNotFound 137 | } 138 | 139 | return data, nil 140 | } 141 | 142 | // Get - fetches the value of the specified k 143 | func (ldb *LevelDB) Get(k string) ([]byte, error) { 144 | return ldb.get(k) 145 | } 146 | 147 | // MGet - fetch multiple values of the specified keys 148 | func (ldb *LevelDB) MGet(keys []string) [][]byte { 149 | var data [][]byte 150 | for _, key := range keys { 151 | val, err := ldb.get(key) 152 | if err != nil { 153 | data = append(data, []byte{}) 154 | continue 155 | } 156 | data = append(data, val) 157 | } 158 | return data 159 | } 160 | 161 | // TTL - returns the time to live of the specified key's value 162 | func (ldb *LevelDB) TTL(key string) int64 { 163 | item, err := ldb.db.Get([]byte(key), nil) 164 | if err != nil { 165 | return -2 166 | } 167 | 168 | parts := bytes.SplitN(item, []byte(expSeparator), 2) 169 | exp, _ := strconv.Atoi(string(parts[0])) 170 | if exp == 0 { 171 | return -1 172 | } 173 | 174 | now := time.Now().Unix() 175 | if now >= int64(exp) { 176 | return -2 177 | } 178 | 179 | return int64(exp) - now 180 | } 181 | 182 | // MDel - removes key(s) from the store 183 | func (ldb *LevelDB) MDel(keys []string) error { 184 | batch := new(leveldb.Batch) 185 | for _, key := range keys { 186 | batch.Delete([]byte(key)) 187 | } 188 | return ldb.db.Write(batch, nil) 189 | } 190 | 191 | // Del - removes key from the store 192 | func (ldb *LevelDB) Del(key string) error { 193 | return ldb.db.Delete([]byte(key), nil) 194 | } 195 | 196 | // Scan - iterate over the whole store using the handler function 197 | func (ldb *LevelDB) Scan(scannerOpt ScannerOptions) error { 198 | var iter iterator.Iterator 199 | 200 | if scannerOpt.Offset == "" { 201 | iter = ldb.db.NewIterator(nil, nil) 202 | } else { 203 | iter = ldb.db.NewIterator(&util.Range{Start: []byte(scannerOpt.Offset)}, nil) 204 | if !scannerOpt.IncludeOffset { 205 | iter.Next() 206 | } 207 | } 208 | 209 | valid := func(k []byte) bool { 210 | if k == nil { 211 | return false 212 | } 213 | 214 | if scannerOpt.Prefix != "" && !bytes.HasPrefix(k, []byte(scannerOpt.Prefix)) { 215 | return false 216 | } 217 | 218 | return true 219 | } 220 | 221 | for iter.Next() { 222 | key := iter.Key() 223 | val := bytes.SplitN(iter.Value(), []byte(";"), 2)[1] 224 | if !valid(key) || scannerOpt.Handler(key, val) != nil { 225 | break 226 | } 227 | } 228 | 229 | iter.Release() 230 | 231 | return iter.Error() 232 | } 233 | -------------------------------------------------------------------------------- /store/hybrid/hybrid.go: -------------------------------------------------------------------------------- 1 | package hybrid 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "runtime" 7 | "time" 8 | 9 | "github.com/projectdiscovery/hmap/store/cache" 10 | "github.com/projectdiscovery/hmap/store/disk" 11 | fileutil "github.com/projectdiscovery/utils/file" 12 | stringsutil "github.com/projectdiscovery/utils/strings" 13 | ) 14 | 15 | type MapType int 16 | 17 | const ( 18 | Memory MapType = iota 19 | Disk 20 | Hybrid 21 | ) 22 | 23 | type DBType int 24 | 25 | const ( 26 | LevelDB DBType = iota 27 | PogrebDB 28 | BBoltDB 29 | PebbleDB 30 | BuntDB 31 | ) 32 | 33 | type Options struct { 34 | MemoryExpirationTime time.Duration 35 | DiskExpirationTime time.Duration 36 | JanitorTime time.Duration 37 | Type MapType 38 | DBType DBType 39 | MemoryGuardForceDisk bool 40 | MemoryGuard bool 41 | MaxMemorySize int 42 | MemoryGuardTime time.Duration 43 | Path string 44 | Cleanup bool 45 | Name string 46 | // Remove temporary hmap in the temporary folder older than duration 47 | RemoveOlderThan time.Duration 48 | } 49 | 50 | var DefaultOptions = Options{ 51 | Type: Memory, 52 | MemoryExpirationTime: time.Duration(5) * time.Minute, 53 | JanitorTime: time.Duration(1) * time.Minute, 54 | } 55 | 56 | var DefaultMemoryOptions = Options{ 57 | Type: Memory, 58 | } 59 | 60 | var DefaultDiskOptions = Options{ 61 | Type: Disk, 62 | DBType: LevelDB, 63 | Cleanup: true, 64 | RemoveOlderThan: 24 * time.Hour * 2, // at startup removes temporary dbs older than x days 65 | } 66 | 67 | var DefaultHybridOptions = Options{ 68 | Type: Hybrid, 69 | DBType: PogrebDB, 70 | MemoryExpirationTime: time.Duration(5) * time.Minute, 71 | JanitorTime: time.Duration(1) * time.Minute, 72 | } 73 | 74 | type HybridMap struct { 75 | options *Options 76 | memorymap cache.Cache 77 | diskmap disk.DB 78 | diskmapPath string 79 | memoryguard *memoryguard 80 | } 81 | 82 | func New(options Options) (*HybridMap, error) { 83 | executableName := fileutil.ExecutableName() 84 | 85 | // Due to potential system failures, the first operation is removing leftovers older than the defined duration 86 | // if cleanup is true and a max age duration has been defined 87 | if options.Cleanup && options.Path == "" && options.RemoveOlderThan > 0 { 88 | targetCleanupDir := os.TempDir() 89 | tmpFiles, err := os.ReadDir(targetCleanupDir) 90 | if err != nil { 91 | return nil, err 92 | } 93 | now := time.Now() 94 | for _, tmpFile := range tmpFiles { 95 | // discard non folders 96 | if !tmpFile.IsDir() { 97 | continue 98 | } 99 | // discard folders not containing the executable name 100 | if !stringsutil.ContainsAny(tmpFile.Name(), executableName) { 101 | continue 102 | } 103 | 104 | tmpFileInfo, err := tmpFile.Info() 105 | if err != nil { 106 | continue 107 | } 108 | modTime := tmpFileInfo.ModTime() 109 | if now.Sub(modTime) > options.RemoveOlderThan { 110 | targetFolderFullPath := filepath.Join(targetCleanupDir, tmpFileInfo.Name()) 111 | os.RemoveAll(targetFolderFullPath) 112 | } 113 | } 114 | } 115 | 116 | var hm HybridMap 117 | if options.Type == Memory || options.Type == Hybrid { 118 | hm.memorymap = cache.New(options.MemoryExpirationTime, options.JanitorTime) 119 | } 120 | 121 | if options.Type == Disk || options.Type == Hybrid { 122 | diskmapPathm := options.Path 123 | if diskmapPathm == "" { 124 | var err error 125 | diskmapPathm, err = os.MkdirTemp("", executableName) 126 | if err != nil { 127 | return nil, err 128 | } 129 | } 130 | 131 | hm.diskmapPath = diskmapPathm 132 | switch options.DBType { 133 | case PogrebDB: 134 | if disk.OpenPogrebDB == nil { 135 | return nil, disk.ErrNotSupported 136 | } 137 | db, err := disk.OpenPogrebDB(diskmapPathm) 138 | if err != nil { 139 | return nil, err 140 | } 141 | hm.diskmap = db 142 | case BBoltDB: 143 | db, err := disk.OpenBoltDBB(filepath.Join(diskmapPathm, "bb")) 144 | if err != nil { 145 | return nil, err 146 | } 147 | db.BucketName = options.Name 148 | hm.diskmap = db 149 | case BuntDB: 150 | db, err := disk.OpenBuntDB(filepath.Join(diskmapPathm, "bunt")) 151 | if err != nil { 152 | return nil, err 153 | } 154 | hm.diskmap = db 155 | case LevelDB: 156 | fallthrough 157 | default: 158 | db, err := disk.OpenLevelDB(diskmapPathm) 159 | if err != nil { 160 | return nil, err 161 | } 162 | hm.diskmap = db 163 | } 164 | } 165 | 166 | if options.Type == Hybrid { 167 | hm.memorymap.OnEvicted(func(k string, v interface{}) { 168 | _ = hm.diskmap.Set(k, v.([]byte), 0) 169 | }) 170 | } 171 | 172 | if options.MemoryGuard { 173 | runMemoryGuard(&hm, options.MemoryGuardTime) 174 | runtime.SetFinalizer(&hm, stopMemoryGuard) 175 | } 176 | 177 | hm.options = &options 178 | 179 | return &hm, nil 180 | } 181 | 182 | func (hm *HybridMap) Close() error { 183 | if hm.diskmap != (disk.DB)(nil) { 184 | hm.diskmap.Close() 185 | } 186 | if hm.diskmapPath != "" && hm.options.Cleanup { 187 | return os.RemoveAll(hm.diskmapPath) 188 | } 189 | return nil 190 | } 191 | 192 | func (hm *HybridMap) Set(k string, v []byte) error { 193 | var err error 194 | switch hm.options.Type { 195 | case Hybrid: 196 | fallthrough 197 | case Memory: 198 | if hm.options.MemoryGuardForceDisk { 199 | err = hm.diskmap.Set(k, v, hm.options.DiskExpirationTime) 200 | } else { 201 | hm.memorymap.Set(k, v) 202 | } 203 | case Disk: 204 | err = hm.diskmap.Set(k, v, hm.options.DiskExpirationTime) 205 | } 206 | 207 | return err 208 | } 209 | 210 | func (hm *HybridMap) Get(k string) ([]byte, bool) { 211 | switch hm.options.Type { 212 | case Memory: 213 | v, ok := hm.memorymap.Get(k) 214 | if ok { 215 | return v.([]byte), ok 216 | } 217 | return []byte{}, ok 218 | case Hybrid: 219 | v, ok := hm.memorymap.Get(k) 220 | if ok { 221 | return v.([]byte), ok 222 | } 223 | vm, err := hm.diskmap.Get(k) 224 | // load it in memory since it has been recently used 225 | if err == nil { 226 | hm.memorymap.Set(k, vm) 227 | } 228 | return vm, err == nil 229 | case Disk: 230 | v, err := hm.diskmap.Get(k) 231 | return v, err == nil 232 | } 233 | 234 | return []byte{}, false 235 | } 236 | 237 | func (hm *HybridMap) Del(key string) error { 238 | switch hm.options.Type { 239 | case Memory: 240 | hm.memorymap.Delete(key) 241 | case Hybrid: 242 | hm.memorymap.Delete(key) 243 | return hm.diskmap.Del(key) 244 | case Disk: 245 | return hm.diskmap.Del(key) 246 | } 247 | 248 | return nil 249 | } 250 | 251 | func (hm *HybridMap) Scan(f func([]byte, []byte) error) { 252 | switch hm.options.Type { 253 | case Memory: 254 | hm.memorymap.Scan(f) 255 | case Hybrid: 256 | hm.memorymap.Scan(f) 257 | _ = hm.diskmap.Scan(disk.ScannerOptions{Handler: f}) 258 | case Disk: 259 | _ = hm.diskmap.Scan(disk.ScannerOptions{Handler: f}) 260 | } 261 | } 262 | 263 | func (hm *HybridMap) Size() int64 { 264 | var count int64 265 | if hm.memorymap != nil { 266 | count += int64(hm.memorymap.ItemCount()) 267 | } 268 | if hm.diskmap != (disk.DB)(nil) { 269 | count += hm.diskmap.Size() 270 | } 271 | return count 272 | } 273 | 274 | func (hm *HybridMap) TuneMemory() { 275 | // si := sysinfo.Get() 276 | var m runtime.MemStats 277 | runtime.ReadMemStats(&m) 278 | if m.Alloc >= uint64(hm.options.MaxMemorySize) { 279 | hm.options.MemoryGuardForceDisk = true 280 | } else { 281 | hm.options.MemoryGuardForceDisk = false 282 | } 283 | } 284 | -------------------------------------------------------------------------------- /filekv/file.go: -------------------------------------------------------------------------------- 1 | package filekv 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "compress/zlib" 7 | "io" 8 | "os" 9 | "sync" 10 | 11 | "github.com/bits-and-blooms/bloom/v3" 12 | lru "github.com/hashicorp/golang-lru/v2" 13 | fileutil "github.com/projectdiscovery/utils/file" 14 | "github.com/syndtr/goleveldb/leveldb" 15 | ) 16 | 17 | // FileDB - represents a file db implementation 18 | type FileDB struct { 19 | stats Stats 20 | options Options 21 | tmpDbName string 22 | tmpDb *os.File 23 | tmpDbWriter io.WriteCloser 24 | db *os.File 25 | dbWriter io.WriteCloser 26 | 27 | // todo: refactor into independent package 28 | mapdb map[string]struct{} 29 | mdb *lru.Cache[string, struct{}] // lru cache 30 | bdb *bloom.BloomFilter // bloom filter 31 | ddb *leveldb.DB // disk based filter 32 | ddbName string 33 | 34 | sync.RWMutex 35 | } 36 | 37 | // Open a new file based db 38 | func Open(options Options) (*FileDB, error) { 39 | db, err := os.OpenFile(options.Path, os.O_RDWR|os.O_CREATE|os.O_APPEND, os.ModePerm) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | tmpFileName, err := fileutil.GetTempFileName() 45 | if err != nil { 46 | return nil, err 47 | } 48 | tmpDb, err := os.Create(tmpFileName) 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | fdb := &FileDB{ 54 | tmpDbName: tmpFileName, 55 | options: options, 56 | db: db, 57 | tmpDb: tmpDb, 58 | } 59 | 60 | if options.Compress { 61 | fdb.tmpDbWriter = zlib.NewWriter(fdb.tmpDb) 62 | fdb.dbWriter = zlib.NewWriter(fdb.db) 63 | } else { 64 | fdb.tmpDbWriter = fdb.tmpDb 65 | fdb.dbWriter = fdb.db 66 | } 67 | 68 | return fdb, nil 69 | } 70 | 71 | // Process added files/slices/elements 72 | func (fdb *FileDB) Process() error { 73 | // Closes the temporary file 74 | if fdb.options.Compress { 75 | // close the writer 76 | if err := fdb.tmpDbWriter.Close(); err != nil { 77 | return err 78 | } 79 | } 80 | 81 | // closes the file to flush to disk and reopen it 82 | _ = fdb.tmpDb.Close() 83 | var err error 84 | fdb.tmpDb, err = os.Open(fdb.tmpDbName) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | var maxItems uint 90 | switch { 91 | case fdb.options.MaxItems > 0: 92 | maxItems = fdb.options.MaxItems 93 | case fdb.stats.NumberOfAddedItems < MaxItems: 94 | maxItems = fdb.stats.NumberOfAddedItems 95 | default: 96 | maxItems = MaxItems 97 | } 98 | 99 | // size the filter according to the number of input items 100 | switch fdb.options.Dedupe { 101 | case MemoryMap: 102 | fdb.mapdb = make(map[string]struct{}, maxItems) 103 | case MemoryLRU: 104 | fdb.mdb, err = lru.New[string, struct{}](int(maxItems)) 105 | if err != nil { 106 | return err 107 | } 108 | case MemoryFilter: 109 | fdb.bdb = bloom.NewWithEstimates(maxItems, FpRatio) 110 | case DiskFilter: 111 | // using executable name so the same app using hmap will remove the files after a certain amount of time 112 | fdb.ddbName, err = os.MkdirTemp("", fileutil.ExecutableName()) 113 | if err != nil { 114 | return err 115 | } 116 | fdb.ddb, err = leveldb.OpenFile(fdb.ddbName, nil) 117 | if err != nil { 118 | return err 119 | } 120 | } 121 | 122 | var tmpDbReader io.Reader 123 | if fdb.options.Compress { 124 | var err error 125 | tmpDbReader, err = zlib.NewReader(fdb.tmpDb) 126 | if err != nil { 127 | return err 128 | } 129 | } else { 130 | tmpDbReader = fdb.tmpDb 131 | } 132 | 133 | sc := bufio.NewScanner(tmpDbReader) 134 | buf := make([]byte, BufferSize) 135 | sc.Buffer(buf, BufferSize) 136 | for sc.Scan() { 137 | _ = fdb.Set(sc.Bytes(), nil) 138 | } 139 | 140 | fdb.tmpDb.Close() 141 | 142 | // flush to disk 143 | fdb.dbWriter.Close() 144 | fdb.db.Close() 145 | 146 | // cleanup filters 147 | switch fdb.options.Dedupe { 148 | case MemoryMap: 149 | fdb.mapdb = nil 150 | case MemoryLRU: 151 | fdb.mdb.Purge() 152 | case MemoryFilter: 153 | fdb.bdb.ClearAll() 154 | case DiskFilter: 155 | fdb.ddb.Close() 156 | os.RemoveAll(fdb.ddbName) 157 | } 158 | 159 | return nil 160 | } 161 | 162 | // Reset the db 163 | func (fdb *FileDB) Reset() error { 164 | // clear the cache 165 | switch fdb.options.Dedupe { 166 | case MemoryMap: 167 | fdb.mapdb = nil 168 | case MemoryLRU: 169 | fdb.mdb.Purge() 170 | case MemoryFilter: 171 | fdb.bdb.ClearAll() 172 | case DiskFilter: 173 | // close - remove - reopen 174 | fdb.ddb.Close() 175 | os.RemoveAll(fdb.ddbName) 176 | var err error 177 | fdb.ddb, err = leveldb.OpenFile(fdb.ddbName, nil) 178 | if err != nil { 179 | return err 180 | } 181 | } 182 | 183 | // reset the tmp file 184 | fdb.tmpDb.Close() 185 | var err error 186 | fdb.tmpDb, err = os.Create(fdb.tmpDbName) 187 | if err != nil { 188 | return err 189 | } 190 | 191 | // reset the target file 192 | fdb.db.Close() 193 | fdb.db, err = os.Create(fdb.tmpDbName) 194 | if err != nil { 195 | return err 196 | } 197 | 198 | if fdb.options.Compress { 199 | fdb.tmpDbWriter = zlib.NewWriter(fdb.tmpDb) 200 | fdb.dbWriter = zlib.NewWriter(fdb.db) 201 | } else { 202 | fdb.tmpDbWriter = fdb.tmpDb 203 | fdb.dbWriter = fdb.db 204 | } 205 | 206 | return nil 207 | } 208 | 209 | // Size - returns the size of the database in bytes 210 | func (fdb *FileDB) Size() int64 { 211 | osstat, err := fdb.db.Stat() 212 | if err != nil { 213 | return 0 214 | } 215 | return osstat.Size() 216 | } 217 | 218 | // Close ... 219 | func (fdb *FileDB) Close() { 220 | tmpDBFilename := fdb.tmpDb.Name() 221 | _ = fdb.tmpDb.Close() 222 | os.RemoveAll(tmpDBFilename) 223 | 224 | _ = fdb.db.Close() 225 | dbFilename := fdb.db.Name() 226 | if fdb.options.Cleanup { 227 | os.RemoveAll(dbFilename) 228 | } 229 | 230 | if fdb.ddbName != "" { 231 | fdb.ddb.Close() 232 | os.RemoveAll(fdb.ddbName) 233 | } 234 | } 235 | 236 | func (fdb *FileDB) set(k, v []byte) error { 237 | var s bytes.Buffer 238 | s.Write(k) 239 | s.WriteString(Separator) 240 | s.Write(v) 241 | s.WriteString(NewLine) 242 | _, err := fdb.dbWriter.Write(s.Bytes()) 243 | if err != nil { 244 | return err 245 | } 246 | fdb.stats.NumberOfItems++ 247 | return nil 248 | } 249 | 250 | func (fdb *FileDB) Set(k, v []byte) error { 251 | // check for duplicates 252 | switch fdb.options.Dedupe { 253 | case MemoryMap: 254 | if _, ok := fdb.mapdb[string(k)]; ok { 255 | fdb.stats.NumberOfDupedItems++ 256 | return ErrItemExists 257 | } 258 | fdb.mapdb[string(k)] = struct{}{} 259 | case MemoryLRU: 260 | if ok, _ := fdb.mdb.ContainsOrAdd(string(k), struct{}{}); ok { 261 | fdb.stats.NumberOfDupedItems++ 262 | return ErrItemExists 263 | } 264 | case MemoryFilter: 265 | if ok := fdb.bdb.TestOrAdd(k); ok { 266 | fdb.stats.NumberOfDupedItems++ 267 | return ErrItemExists 268 | } 269 | case DiskFilter: 270 | if ok, err := fdb.ddb.Has(k, nil); err == nil && ok { 271 | fdb.stats.NumberOfDupedItems++ 272 | return ErrItemExists 273 | } else if err == nil && !ok { 274 | _ = fdb.ddb.Put(k, []byte{}, nil) 275 | } 276 | } 277 | 278 | if fdb.shouldSkip(k, v) { 279 | fdb.stats.NumberOfFilteredItems++ 280 | return ErrItemFiltered 281 | } 282 | 283 | fdb.stats.NumberOfItems++ 284 | return fdb.set(k, v) 285 | } 286 | 287 | // Scan - iterate over the whole store using the handler function 288 | func (fdb *FileDB) Scan(handler func([]byte, []byte) error) error { 289 | // open the db and scan 290 | dbCopy, err := os.Open(fdb.options.Path) 291 | if err != nil { 292 | return err 293 | } 294 | defer dbCopy.Close() 295 | 296 | var dbReader io.ReadCloser 297 | if fdb.options.Compress { 298 | dbReader, err = zlib.NewReader(dbCopy) 299 | if err != nil { 300 | return err 301 | } 302 | } else { 303 | dbReader = dbCopy 304 | } 305 | 306 | sc := bufio.NewScanner(dbReader) 307 | buf := make([]byte, BufferSize) 308 | sc.Buffer(buf, BufferSize) 309 | for sc.Scan() { 310 | tokens := bytes.SplitN(sc.Bytes(), []byte(Separator), 2) 311 | var k, v []byte 312 | if len(tokens) > 0 { 313 | k = tokens[0] 314 | } 315 | if len(tokens) > 1 { 316 | v = tokens[1] 317 | } 318 | if err := handler(k, v); err != nil { 319 | return err 320 | } 321 | } 322 | return nil 323 | } 324 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= 3 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 4 | github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= 5 | github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= 6 | github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= 7 | github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= 8 | github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= 9 | github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= 10 | github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= 11 | github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= 12 | github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= 13 | github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= 14 | github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= 15 | github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= 16 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 17 | github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= 18 | github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= 19 | github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= 20 | github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= 21 | github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= 22 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 23 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 24 | github.com/bits-and-blooms/bitset v1.3.1 h1:y+qrlmq3XsWi+xZqSaueaE8ry8Y127iMxlMfqcK8p0g= 25 | github.com/bits-and-blooms/bitset v1.3.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= 26 | github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= 27 | github.com/bits-and-blooms/bloom/v3 v3.3.1/go.mod h1:bhUUknWd5khVbTe4UgMCSiOOVJzr3tMoijSK3WwvW90= 28 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 29 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 30 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 31 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 32 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 33 | github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= 34 | github.com/cockroachdb/datadriven v1.0.1-0.20211007161720-b558070c3be0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= 35 | github.com/cockroachdb/datadriven v1.0.1-0.20220214170620-9913f5bc19b7/go.mod h1:hi0MtSY3AYDQNDi83kDkMH5/yqM/CsIrsOITkSoH7KI= 36 | github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= 37 | github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= 38 | github.com/cockroachdb/errors v1.8.8/go.mod h1:z6VnEL3hZ/2ONZEvG7S5Ym0bU2AqPcEKnIiA1wbsSu0= 39 | github.com/cockroachdb/errors v1.9.0 h1:B48dYem5SlAY7iU8AKsgedb4gH6mo+bDkbtLIvM/a88= 40 | github.com/cockroachdb/errors v1.9.0/go.mod h1:vaNcEYYqbIqB5JhKBhFV9CneUqeuEbB2OYJBK4GBNYQ= 41 | github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= 42 | github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= 43 | github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= 44 | github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b h1:6TtBvyITBhpkL8EvEEZdS9om8egSxP/mUML01aM3pyY= 45 | github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b/go.mod h1:JsehdjcR1QgLZkqBeYrbVdE3cdxbdrycA/PN+Cg+RNw= 46 | github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= 47 | github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= 48 | github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= 49 | github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= 50 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 51 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 52 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 53 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= 54 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 55 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 56 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 57 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 58 | github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= 59 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 60 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= 61 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 62 | github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= 63 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 64 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 65 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 66 | github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= 67 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 68 | github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= 69 | github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= 70 | github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= 71 | github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= 72 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 73 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 74 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 75 | github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= 76 | github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= 77 | github.com/getsentry/sentry-go v0.16.0 h1:owk+S+5XcgJLlGR/3+3s6N4d+uKwqYvh/eS0AIMjPWo= 78 | github.com/getsentry/sentry-go v0.16.0/go.mod h1:ZXCloQLj0pG7mja5NK6NPf2V4A88YJ4pNlc2mOHwh6Y= 79 | github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= 80 | github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= 81 | github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= 82 | github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= 83 | github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= 84 | github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= 85 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= 86 | github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= 87 | github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= 88 | github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= 89 | github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= 90 | github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= 91 | github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 92 | github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= 93 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 94 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 95 | github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= 96 | github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= 97 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 98 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 99 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 100 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 101 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 102 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 103 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= 104 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 105 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 106 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 107 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 108 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 109 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 110 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 111 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 112 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= 113 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 114 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 115 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 116 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 117 | github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= 118 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 119 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 120 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 121 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 122 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 123 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 124 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 125 | github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 126 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 127 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 128 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 129 | github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= 130 | github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= 131 | github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 132 | github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 133 | github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 134 | github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= 135 | github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= 136 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 137 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 138 | github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= 139 | github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= 140 | github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= 141 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 142 | github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= 143 | github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= 144 | github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= 145 | github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= 146 | github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= 147 | github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= 148 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 149 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 150 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 151 | github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= 152 | github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= 153 | github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= 154 | github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= 155 | github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= 156 | github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= 157 | github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= 158 | github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= 159 | github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= 160 | github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= 161 | github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= 162 | github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= 163 | github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= 164 | github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= 165 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 166 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 167 | github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 168 | github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 169 | github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 170 | github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= 171 | github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= 172 | github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 173 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 174 | github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= 175 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 176 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 177 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 178 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 179 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 180 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 181 | github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= 182 | github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= 183 | github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= 184 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 185 | github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 186 | github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 187 | github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= 188 | github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 189 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 190 | github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= 191 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 192 | github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 193 | github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= 194 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 195 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 196 | github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= 197 | github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= 198 | github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= 199 | github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= 200 | github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= 201 | github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= 202 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 203 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 204 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 205 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 206 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 207 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 208 | github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= 209 | github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= 210 | github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= 211 | github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= 212 | github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= 213 | github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= 214 | github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= 215 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 216 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 217 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= 218 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 219 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 220 | github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 221 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 222 | github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= 223 | github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= 224 | github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= 225 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 226 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 227 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 228 | github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= 229 | github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= 230 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 231 | github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= 232 | github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= 233 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 234 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 235 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 236 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 237 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 238 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 239 | github.com/projectdiscovery/utils v0.0.7 h1:jqDuZedy3t66o6ejQUXjgNWbyAHqiBqLAUDkst9DA2M= 240 | github.com/projectdiscovery/utils v0.0.7/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= 241 | github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= 242 | github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= 243 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 244 | github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= 245 | github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= 246 | github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= 247 | github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= 248 | github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= 249 | github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= 250 | github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 251 | github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= 252 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 253 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 254 | github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= 255 | github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= 256 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 257 | github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 258 | github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= 259 | github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= 260 | github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= 261 | github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= 262 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 263 | github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= 264 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 265 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 266 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 267 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 268 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 269 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= 270 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 271 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 272 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 273 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 274 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 275 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 276 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 277 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 278 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 279 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 280 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 281 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 282 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 283 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 284 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 285 | github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= 286 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 287 | github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= 288 | github.com/tidwall/btree v1.4.3 h1:Lf5U/66bk0ftNppOBjVoy/AIPBrLMkheBp4NnSNiYOo= 289 | github.com/tidwall/btree v1.4.3/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= 290 | github.com/tidwall/buntdb v1.2.10 h1:U/ebfkmYPBnyiNZIirUiWFcxA/mgzjbKlyPynFsPtyM= 291 | github.com/tidwall/buntdb v1.2.10/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= 292 | github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 293 | github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= 294 | github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 295 | github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= 296 | github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= 297 | github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= 298 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 299 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 300 | github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= 301 | github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 302 | github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= 303 | github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= 304 | github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= 305 | github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= 306 | github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= 307 | github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= 308 | github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= 309 | github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= 310 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 311 | github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= 312 | github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= 313 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 314 | github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= 315 | github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= 316 | github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= 317 | github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= 318 | github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= 319 | github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= 320 | github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= 321 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 322 | github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= 323 | github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= 324 | github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= 325 | github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= 326 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 327 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 328 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 329 | go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= 330 | go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= 331 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 332 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 333 | golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 334 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 335 | golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 336 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 337 | golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= 338 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 339 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 340 | golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws= 341 | golang.org/x/exp v0.0.0-20221230185412-738e83a70c30/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= 342 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 343 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 344 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 345 | golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= 346 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 347 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 348 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 349 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 350 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 351 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 352 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 353 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 354 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 355 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 356 | golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 357 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 358 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 359 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 360 | golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 361 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 362 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 363 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 364 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 365 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 366 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 367 | golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= 368 | golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 369 | golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= 370 | golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= 371 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 372 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 373 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 374 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 375 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 376 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 377 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 378 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 379 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 380 | golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= 381 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 382 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 383 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 384 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 385 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 386 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 387 | golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 388 | golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 389 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 390 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 391 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 392 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 393 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 394 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 395 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 396 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 397 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 398 | golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 399 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 400 | golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 401 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 402 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 403 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 404 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 405 | golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 406 | golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 407 | golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 408 | golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= 409 | golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 410 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 411 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 412 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 413 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 414 | golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 415 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 416 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 417 | golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= 418 | golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 419 | golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 420 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 421 | golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 422 | golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 423 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 424 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 425 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 426 | golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 427 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 428 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 429 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 430 | golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 431 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 432 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 433 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 434 | golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 435 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 436 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 437 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 438 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 439 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 440 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 441 | google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 442 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 443 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 444 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 445 | google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= 446 | google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= 447 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 448 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 449 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 450 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 451 | google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= 452 | google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= 453 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 454 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 455 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 456 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 457 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 458 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 459 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 460 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 461 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 462 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 463 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 464 | google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= 465 | google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 466 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 467 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 468 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 469 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 470 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 471 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 472 | gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= 473 | gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= 474 | gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 475 | gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= 476 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 477 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 478 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 479 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 480 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 481 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 482 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 483 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 484 | gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 485 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 486 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 487 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 488 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 489 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 490 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 491 | --------------------------------------------------------------------------------