├── .github ├── release-drafter-config.yml └── workflows │ ├── codeql-analysis.yml │ ├── integration.yml │ └── release-drafter.yml ├── .gitignore ├── .golangci.yml ├── LICENSE ├── Makefile ├── README.md ├── client.go ├── client_test.go ├── example_client_test.go ├── go.mod ├── go.sum ├── pool.go └── pool_test.go /.github/release-drafter-config.yml: -------------------------------------------------------------------------------- 1 | name-template: 'Version $NEXT_PATCH_VERSION' 2 | tag-template: 'v$NEXT_PATCH_VERSION' 3 | categories: 4 | - title: 'Features' 5 | labels: 6 | - 'feature' 7 | - 'enhancement' 8 | - title: 'Bug Fixes' 9 | labels: 10 | - 'fix' 11 | - 'bugfix' 12 | - 'bug' 13 | - title: 'Maintenance' 14 | label: 'chore' 15 | change-template: '- $TITLE (#$NUMBER)' 16 | exclude-labels: 17 | - 'skip-changelog' 18 | template: | 19 | ## Changes 20 | 21 | $CHANGES 22 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '18 13 * * 0' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | 28 | strategy: 29 | fail-fast: false 30 | matrix: 31 | language: [ 'go' ] 32 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 33 | # Learn more: 34 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 35 | 36 | steps: 37 | - name: Checkout repository 38 | uses: actions/checkout@v2 39 | 40 | # Initializes the CodeQL tools for scanning. 41 | - name: Initialize CodeQL 42 | uses: github/codeql-action/init@v1 43 | with: 44 | languages: ${{ matrix.language }} 45 | # If you wish to specify custom queries, you can do so here or in a config file. 46 | # By default, queries listed here will override any specified in a config file. 47 | # Prefix the list here with "+" to use these queries and those in the config file. 48 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 49 | 50 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 51 | # If this step fails, then you should remove it and run the build manually (see below) 52 | - name: Autobuild 53 | uses: github/codeql-action/autobuild@v1 54 | 55 | # ℹ️ Command-line programs to run using the OS shell. 56 | # 📚 https://git.io/JvXDl 57 | 58 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 59 | # and modify them (or add more) to build your code if your project 60 | # uses a compiled language 61 | 62 | #- run: | 63 | # make bootstrap 64 | # make release 65 | 66 | - name: Perform CodeQL Analysis 67 | uses: github/codeql-action/analyze@v1 68 | -------------------------------------------------------------------------------- /.github/workflows/integration.yml: -------------------------------------------------------------------------------- 1 | name: Integration 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - 'docs/**' 7 | - '**/*.rst' 8 | - '**/*.md' 9 | branches: 10 | - master 11 | - main 12 | - '[0-9].[0-9]' 13 | pull_request: 14 | branches: 15 | - master 16 | - main 17 | - '[0-9].[0-9]' 18 | schedule: 19 | - cron: '0 1 * * *' 20 | 21 | jobs: 22 | 23 | lint: 24 | name: Code linters 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v3 28 | - run: | 29 | make checkfmt 30 | 31 | integration: 32 | name: Build and test 33 | runs-on: ubuntu-latest 34 | steps: 35 | - uses: actions/setup-go@v3 36 | with: 37 | go-version: 1.18.x 38 | - uses: actions/checkout@v3 39 | - run: docker run -p 6379:6379 -d redis/redis-stack-server:edge 40 | - run: | 41 | make get 42 | make coverage 43 | - name: Upload coverage 44 | uses: codecov/codecov-action@v3 45 | -------------------------------------------------------------------------------- /.github/workflows/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name: Release Drafter 2 | 3 | on: 4 | push: 5 | # branches to consider in the event; optional, defaults to all 6 | branches: 7 | - master 8 | 9 | jobs: 10 | update_release_draft: 11 | runs-on: ubuntu-latest 12 | steps: 13 | # Drafts your next Release notes as Pull Requests are merged into "master" 14 | - uses: release-drafter/release-drafter@v5 15 | with: 16 | # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml 17 | config-name: release-drafter-config.yml 18 | env: 19 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | coverage.txt 14 | 15 | .idea 16 | .project 17 | 18 | vendor/ 19 | 20 | # OS generated files # 21 | ###################### 22 | .DS_Store 23 | .DS_Store? 24 | ._* 25 | .Spotlight-V100 26 | .Trashes 27 | ehthumbs.db 28 | Thumbs.db -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # options for analysis running 2 | run: 3 | # include test files or not, default is true 4 | tests: false 5 | 6 | linters-settings: 7 | golint: 8 | # minimal confidence for issues, default is 0.8 9 | min-confidence: 0.8 10 | 11 | exclude-rules: 12 | # Exclude some linters from running on tests files. 13 | - path: _test\.go 14 | linters: 15 | - errcheck -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2020, RedisBloom 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Go parameters 2 | GOCMD=GO111MODULE=on go 3 | GOBUILD=$(GOCMD) build 4 | GOINSTALL=$(GOCMD) install 5 | GOCLEAN=$(GOCMD) clean 6 | GOTEST=$(GOCMD) test 7 | GOGET=$(GOCMD) get 8 | GOMOD=$(GOCMD) mod 9 | GOFMT=$(GOCMD) fmt 10 | 11 | .PHONY: all test coverage 12 | all: test coverage 13 | 14 | checkfmt: 15 | @echo 'Checking gofmt';\ 16 | bash -c "diff -u <(echo -n) <(gofmt -d .)";\ 17 | EXIT_CODE=$$?;\ 18 | if [ "$$EXIT_CODE" -ne 0 ]; then \ 19 | echo '$@: Go files must be formatted with go fmt. Please run `make fmt`'; \ 20 | fi && \ 21 | exit $$EXIT_CODE 22 | 23 | fmt: 24 | $(GOFMT) ./... 25 | 26 | get: 27 | $(GOGET) -t -v ./... 28 | 29 | test: get 30 | $(GOFMT) ./... 31 | $(GOTEST) -count 1 ./... 32 | 33 | coverage: get test 34 | $(GOTEST) -race -coverprofile=coverage.txt -covermode=atomic . 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![license](https://img.shields.io/github/license/RedisBloom/redisbloom-go.svg)](https://github.com/RedisBloom/redisbloom-go) 2 | [![GitHub issues](https://img.shields.io/github/release/RedisBloom/redisbloom-go.svg)](https://github.com/RedisBloom/redisbloom-go/releases/latest) 3 | [![Codecov](https://codecov.io/gh/RedisBloom/redisbloom-go/branch/master/graph/badge.svg)](https://codecov.io/gh/RedisBloom/redisbloom-go) 4 | [![GoDoc](https://godoc.org/github.com/RedisBloom/redisbloom-go?status.svg)](https://pkg.go.dev/github.com/RedisBloom/redisbloom-go) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/RedisBloom/redisbloom-go)](https://goreportcard.com/report/github.com/RedisBloom/redisbloom-go) 6 | 7 | # redisbloom-go 8 | [![Forum](https://img.shields.io/badge/Forum-RedisBloom-blue)](https://forum.redislabs.com/c/modules/redisbloom) 9 | [![Discord](https://img.shields.io/discord/697882427875393627?style=flat-square)](https://discord.gg/wXhwjCQ) 10 | 11 | Go client for RedisBloom (https://github.com/RedisBloom/redisbloom), based on redigo. 12 | 13 | ## Installing 14 | 15 | ```sh 16 | $ go get github.com/RedisBloom/redisbloom-go 17 | ``` 18 | 19 | ## Running tests 20 | 21 | A simple test suite is provided, and can be run with: 22 | 23 | ```sh 24 | $ go test 25 | ``` 26 | 27 | The tests expect a Redis server with the RedisBloom module loaded to be available at localhost:6379. You can easily launch RedisBloom with Docker in the following manner: 28 | ``` 29 | docker run -d -p 6379:6379 --name redis-redisbloom redis/redis-stack-server:latest 30 | ``` 31 | 32 | ## Example Code 33 | 34 | Make sure to check the full list of examples at [Pkg.go.dev](https://pkg.go.dev/github.com/RedisBloom/redisbloom-go#pkg-examples). 35 | 36 | ```go 37 | package main 38 | 39 | import ( 40 | "fmt" 41 | redisbloom "github.com/RedisBloom/redisbloom-go" 42 | ) 43 | 44 | func main() { 45 | // Connect to localhost with no password 46 | var client = redisbloom.NewClient("localhost:6379", "nohelp", nil) 47 | 48 | // BF.ADD mytest item 49 | _, err := client.Add("mytest", "myItem") 50 | if err != nil { 51 | fmt.Println("Error:", err) 52 | } 53 | 54 | exists, err := client.Exists("mytest", "myItem") 55 | if err != nil { 56 | fmt.Println("Error:", err) 57 | } 58 | fmt.Println("myItem exists in mytest: ", exists) 59 | } 60 | ``` 61 | 62 | ## Supported RedisBloom Commands 63 | 64 | Make sure to check the full command reference at [redisbloom.io](https://redisbloom.io). 65 | 66 | ### Bloom Filter 67 | 68 | | Command | Recommended API and godoc | 69 | | :--- | ----: | 70 | | [BF.RESERVE](https://oss.redislabs.com/redisbloom/Bloom_Commands/#bfreserve) | [Reserve](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.Reserve) | 71 | | [BF.ADD](https://oss.redislabs.com/redisbloom/Bloom_Commands/#bfadd) | [Add](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.Add) | 72 | | [BF.MADD](https://oss.redislabs.com/redisbloom/Bloom_Commands/#bfmadd) | [BfAddMulti](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.BfAddMulti) | 73 | | [BF.INSERT](https://oss.redislabs.com/redisbloom/Bloom_Commands/#bfinsert) | [BfInsert](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.BfInsert) | 74 | | [BF.EXISTS](https://oss.redislabs.com/redisbloom/Bloom_Commands/#bfexists) | [Exists](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.Exists) | 75 | | [BF.MEXISTS](https://oss.redislabs.com/redisbloom/Bloom_Commands/#bfmexists) | [BfExistsMulti](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.BfExistsMulti) | 76 | | [BF.SCANDUMP](https://oss.redislabs.com/redisbloom/Bloom_Commands/#bfscandump) | [BfScanDump](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.BfScanDump) | 77 | | [BF.LOADCHUNK](https://oss.redislabs.com/redisbloom/Bloom_Commands/#bfloadchunk) | [BfLoadChunk](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.BfLoadChunk) | 78 | | [BF.INFO](https://oss.redislabs.com/redisbloom/Bloom_Commands/#bfinfo) | [Info](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.Info) | 79 | 80 | ### Cuckoo Filter 81 | 82 | | Command | Recommended API and godoc | 83 | | :--- | ----: | 84 | | [CF.RESERVE](https://oss.redislabs.com/redisbloom/Cuckoo_Commands/#cfreserve) | [CfReserve](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CfReserve) | 85 | | [CF.ADD](https://oss.redislabs.com/redisbloom/Cuckoo_Commands/#cfadd) | [CfAdd](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CfAdd) | 86 | | [CF.ADDNX](https://oss.redislabs.com/redisbloom/Cuckoo_Commands/#cfaddnx) | [CfAddNx](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CfAddNx) | 87 | | [CF.INSERT](https://oss.redislabs.com/redisbloom/Cuckoo_Commands/#cfinsert) | [CfInsert](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CfInsert) | 88 | | [CF.INSERTNX](https://oss.redislabs.com/redisbloom/Cuckoo_Commands/#cfinsertnx) | [CfInsertNx](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CfInsertNx) | 89 | | [CF.EXISTS](https://oss.redislabs.com/redisbloom/Cuckoo_Commands/#cfexists) | [CfExists](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CfExists) | 90 | | [CF.DEL](https://oss.redislabs.com/redisbloom/Cuckoo_Commands/#cfdel) | [CfDel](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CfDel) | 91 | | [CF.COUNT](https://oss.redislabs.com/redisbloom/Cuckoo_Commands/#cfcount) | [CfCount](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CfCount) | 92 | | [CF.SCANDUMP](https://oss.redislabs.com/redisbloom/Cuckoo_Commands/#cfscandump) | [CfScanDump](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CfScanDump) | 93 | | [CF.LOADCHUNK](https://oss.redislabs.com/redisbloom/Cuckoo_Commands/#cfloadchunck) | [CfLoadChunk](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CfLoadChunk) | 94 | | [CF.INFO](https://oss.redislabs.com/redisbloom/Cuckoo_Commands/#cfinfo) | [CfInfo](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CfInfo) | 95 | 96 | ### Count-Min Sketch 97 | 98 | | Command | Recommended API and godoc | 99 | | :--- | ----: | 100 | | [CMS.INITBYDIM](https://oss.redislabs.com/redisbloom/CountMinSketch_Commands/#cmsinitbydim) | [CmsInitByDim](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CmsInitByDim) | 101 | | [CMS.INITBYPROB](https://oss.redislabs.com/redisbloom/CountMinSketch_Commands/#cmsinitbyprob) | [CmsInitByProb](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CmsInitByProb) | 102 | | [CMS.INCRBY](https://oss.redislabs.com/redisbloom/CountMinSketch_Commands/#cmsincrby) | [CmsIncrBy](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CmsIncrBy) | 103 | | [CMS.QUERY](https://oss.redislabs.com/redisbloom/CountMinSketch_Commands/#cmsquery) | [CmsQuery](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CmsQuery) | 104 | | [CMS.MERGE](https://oss.redislabs.com/redisbloom/CountMinSketch_Commands/#cmsmerge) | [CmsMerge](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CmsMerge) | 105 | | [CMS.INFO](https://oss.redislabs.com/redisbloom/CountMinSketch_Commands/#cmsinfo) | [CmsInfo](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.CmsInfo) | 106 | 107 | ### TopK Filter 108 | 109 | | Command | Recommended API and godoc | 110 | | :--- | ----: | 111 | | [TOPK.RESERVE](https://oss.redislabs.com/redisbloom/TopK_Commands/#topkreserve) | [TopkReserve](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.TopkReserve) | 112 | | [TOPK.ADD](https://oss.redislabs.com/redisbloom/TopK_Commands/#topkadd) | [TopkAdd](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.TopkAdd) | 113 | | [TOPK.INCRBY](https://oss.redislabs.com/redisbloom/TopK_Commands/#topkincrby) | [TopkIncrby](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.TopkIncrby) | 114 | | [TOPK.QUERY](https://oss.redislabs.com/redisbloom/TopK_Commands/#topkquery) | [TopkQuery](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.TopkQuery) | 115 | | [TOPK.COUNT](https://oss.redislabs.com/redisbloom/TopK_Commands/#topkcount) | [TopkCount](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.TopkCount) | 116 | | [TOPK.LIST](https://oss.redislabs.com/redisbloom/TopK_Commands/#topklist) | [TopkList](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.TopkList) | 117 | | [TOPK.INFO](https://oss.redislabs.com/redisbloom/TopK_Commands/#topkinfo) | [TopkInfo](https://godoc.org/github.com/RedisBloom/redisbloom-go#Client.TopkInfo) | 118 | 119 | 120 | ## License 121 | 122 | redisbloom-go is distributed under the BSD 3-Clause license - see [LICENSE](LICENSE) 123 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package redis_bloom_go 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | 9 | "github.com/gomodule/redigo/redis" 10 | ) 11 | 12 | // TODO: refactor this hard limit and revise client locking 13 | // Client Max Connections 14 | var maxConns = 500 15 | 16 | // Client is an interface to RedisBloom redis commands 17 | type Client struct { 18 | Pool ConnPool 19 | Name string 20 | } 21 | 22 | // TDigestInfo is a struct that represents T-Digest properties 23 | type TDigestInfo struct { 24 | compression int64 25 | capacity int64 26 | mergedNodes int64 27 | unmergedNodes int64 28 | mergedWeight int64 29 | unmergedWeight int64 30 | totalCompressions int64 31 | } 32 | 33 | // Compression - returns the compression of TDigestInfo instance 34 | func (info *TDigestInfo) Compression() int64 { 35 | return info.compression 36 | } 37 | 38 | // Capacity - returns the capacity of TDigestInfo instance 39 | func (info *TDigestInfo) Capacity() int64 { 40 | return info.capacity 41 | } 42 | 43 | // MergedNodes - returns the merged nodes of TDigestInfo instance 44 | func (info *TDigestInfo) MergedNodes() int64 { 45 | return info.mergedNodes 46 | } 47 | 48 | // UnmergedNodes - returns the unmerged nodes of TDigestInfo instance 49 | func (info *TDigestInfo) UnmergedNodes() int64 { 50 | return info.unmergedNodes 51 | } 52 | 53 | // MergedWeight - returns the merged weight of TDigestInfo instance 54 | func (info *TDigestInfo) MergedWeight() int64 { 55 | return info.mergedWeight 56 | } 57 | 58 | // UnmergedWeight - returns the unmerged weight of TDigestInfo instance 59 | func (info *TDigestInfo) UnmergedWeight() int64 { 60 | return info.unmergedWeight 61 | } 62 | 63 | // TotalCompressions - returns the total compressions of TDigestInfo instance 64 | func (info *TDigestInfo) TotalCompressions() int64 { 65 | return info.totalCompressions 66 | } 67 | 68 | // NewClient creates a new client connecting to the redis host, and using the given name as key prefix. 69 | // Addr can be a single host:port pair, or a comma separated list of host:port,host:port... 70 | // In the case of multiple hosts we create a multi-pool and select connections at random 71 | // Deprecated: Please use NewClientFromPool() instead 72 | func NewClient(addr, name string, authPass *string) *Client { 73 | addrs := strings.Split(addr, ",") 74 | var pool ConnPool 75 | if len(addrs) == 1 { 76 | pool = NewSingleHostPool(addrs[0], authPass) 77 | } else { 78 | pool = NewMultiHostPool(addrs, authPass) 79 | } 80 | ret := &Client{ 81 | Pool: pool, 82 | Name: name, 83 | } 84 | return ret 85 | } 86 | 87 | // NewClientFromPool creates a new Client with the given pool and client name 88 | func NewClientFromPool(pool *redis.Pool, name string) *Client { 89 | ret := &Client{ 90 | Pool: pool, 91 | Name: name, 92 | } 93 | return ret 94 | } 95 | 96 | // Reserve - Creates an empty Bloom Filter with a given desired error ratio and initial capacity. 97 | // args: 98 | // key - the name of the filter 99 | // error_rate - the desired probability for false positives 100 | // capacity - the number of entries you intend to add to the filter 101 | func (client *Client) Reserve(key string, error_rate float64, capacity uint64) (err error) { 102 | conn := client.Pool.Get() 103 | defer conn.Close() 104 | _, err = conn.Do("BF.RESERVE", key, strconv.FormatFloat(error_rate, 'g', 16, 64), capacity) 105 | return err 106 | } 107 | 108 | // Add - Add (or create and add) a new value to the filter 109 | // args: 110 | // key - the name of the filter 111 | // item - the item to add 112 | func (client *Client) Add(key string, item string) (exists bool, err error) { 113 | conn := client.Pool.Get() 114 | defer conn.Close() 115 | return redis.Bool(conn.Do("BF.ADD", key, item)) 116 | } 117 | 118 | // Exists - Determines whether an item may exist in the Bloom Filter or not. 119 | // args: 120 | // key - the name of the filter 121 | // item - the item to check for 122 | func (client *Client) Exists(key string, item string) (exists bool, err error) { 123 | conn := client.Pool.Get() 124 | defer conn.Close() 125 | return redis.Bool(conn.Do("BF.EXISTS", key, item)) 126 | } 127 | 128 | // Info - Return information about key 129 | // args: 130 | // key - the name of the filter 131 | func (client *Client) Info(key string) (info map[string]int64, err error) { 132 | conn := client.Pool.Get() 133 | defer conn.Close() 134 | result, err := conn.Do("BF.INFO", key) 135 | if err != nil { 136 | return nil, err 137 | } 138 | 139 | values, err := redis.Values(result, nil) 140 | if err != nil { 141 | return nil, err 142 | } 143 | if len(values)%2 != 0 { 144 | return nil, errors.New("Info expects even number of values result") 145 | } 146 | info = map[string]int64{} 147 | for i := 0; i < len(values); i += 2 { 148 | key, err = redis.String(values[i], nil) 149 | if err != nil { 150 | return nil, err 151 | } 152 | info[key], err = redis.Int64(values[i+1], nil) 153 | if err != nil { 154 | return nil, err 155 | } 156 | } 157 | return info, nil 158 | } 159 | 160 | // BfAddMulti - Adds one or more items to the Bloom Filter, creating the filter if it does not yet exist. 161 | // args: 162 | // key - the name of the filter 163 | // item - One or more items to add 164 | func (client *Client) BfAddMulti(key string, items []string) ([]int64, error) { 165 | conn := client.Pool.Get() 166 | defer conn.Close() 167 | args := redis.Args{key}.AddFlat(items) 168 | result, err := conn.Do("BF.MADD", args...) 169 | return redis.Int64s(result, err) 170 | } 171 | 172 | func (client *Client) BfCard(key string) (int64, error) { 173 | conn := client.Pool.Get() 174 | defer conn.Close() 175 | args := redis.Args{key} 176 | result, err := conn.Do("BF.CARD", args...) 177 | return redis.Int64(result, err) 178 | } 179 | 180 | // BfExistsMulti - Determines if one or more items may exist in the filter or not. 181 | // args: 182 | // key - the name of the filter 183 | // item - one or more items to check 184 | func (client *Client) BfExistsMulti(key string, items []string) ([]int64, error) { 185 | conn := client.Pool.Get() 186 | defer conn.Close() 187 | args := redis.Args{key}.AddFlat(items) 188 | result, err := conn.Do("BF.MEXISTS", args...) 189 | return redis.Int64s(result, err) 190 | } 191 | 192 | // Begins an incremental save of the bloom filter. 193 | func (client *Client) BfScanDump(key string, iter int64) (int64, []byte, error) { 194 | conn := client.Pool.Get() 195 | defer conn.Close() 196 | reply, err := redis.Values(conn.Do("BF.SCANDUMP", key, iter)) 197 | if err != nil || len(reply) != 2 { 198 | return 0, nil, err 199 | } 200 | iter = reply[0].(int64) 201 | if reply[1] == nil { 202 | return iter, nil, err 203 | } 204 | return iter, reply[1].([]byte), err 205 | } 206 | 207 | // Restores a filter previously saved using SCANDUMP . 208 | func (client *Client) BfLoadChunk(key string, iter int64, data []byte) (string, error) { 209 | conn := client.Pool.Get() 210 | defer conn.Close() 211 | return redis.String(conn.Do("BF.LOADCHUNK", key, iter, data)) 212 | } 213 | 214 | // This command will add one or more items to the bloom filter, by default creating it if it does not yet exist. 215 | func (client *Client) BfInsert(key string, cap int64, errorRatio float64, expansion int64, noCreate bool, nonScaling bool, items []string) (res []int64, err error) { 216 | conn := client.Pool.Get() 217 | defer conn.Close() 218 | args := redis.Args{key} 219 | if cap > 0 { 220 | args = args.Add("CAPACITY", cap) 221 | } 222 | if errorRatio > 0 { 223 | args = args.Add("ERROR", errorRatio) 224 | } 225 | if expansion > 0 { 226 | args = args.Add("EXPANSION", expansion) 227 | } 228 | if noCreate { 229 | args = args.Add("NOCREATE") 230 | } 231 | if nonScaling { 232 | args = args.Add("NONSCALING") 233 | } 234 | args = args.Add("ITEMS").AddFlat(items) 235 | var resp []interface{} 236 | var innerRes int64 237 | resp, err = redis.Values(conn.Do("BF.INSERT", args...)) 238 | if err != nil { 239 | return 240 | } 241 | for _, arrayPos := range resp { 242 | innerRes, err = redis.Int64(arrayPos, err) 243 | if err == nil { 244 | res = append(res, innerRes) 245 | } else { 246 | break 247 | } 248 | } 249 | return 250 | } 251 | 252 | // Initializes a TopK with specified parameters. 253 | func (client *Client) TopkReserve(key string, topk int64, width int64, depth int64, decay float64) (string, error) { 254 | conn := client.Pool.Get() 255 | defer conn.Close() 256 | result, err := conn.Do("TOPK.RESERVE", key, topk, width, depth, strconv.FormatFloat(decay, 'g', 16, 64)) 257 | return redis.String(result, err) 258 | } 259 | 260 | // Adds an item to the data structure. 261 | func (client *Client) TopkAdd(key string, items []string) ([]string, error) { 262 | conn := client.Pool.Get() 263 | defer conn.Close() 264 | args := redis.Args{key}.AddFlat(items) 265 | result, err := conn.Do("TOPK.ADD", args...) 266 | return redis.Strings(result, err) 267 | } 268 | 269 | // Returns count for an item. 270 | func (client *Client) TopkCount(key string, items []string) (result []int64, err error) { 271 | conn := client.Pool.Get() 272 | defer conn.Close() 273 | args := redis.Args{key}.AddFlat(items) 274 | result, err = redis.Int64s(conn.Do("TOPK.COUNT", args...)) 275 | return 276 | } 277 | 278 | // Checks whether an item is one of Top-K items. 279 | func (client *Client) TopkQuery(key string, items []string) ([]int64, error) { 280 | conn := client.Pool.Get() 281 | defer conn.Close() 282 | args := redis.Args{key}.AddFlat(items) 283 | result, err := conn.Do("TOPK.QUERY", args...) 284 | return redis.Int64s(result, err) 285 | } 286 | 287 | // Return full list of items in Top K list. 288 | func (client *Client) TopkListWithCount(key string) (map[string]int64, error) { 289 | conn := client.Pool.Get() 290 | defer conn.Close() 291 | return ParseInfoReply(redis.Values(conn.Do("TOPK.LIST", key, "WITHCOUNT"))) 292 | } 293 | 294 | func (client *Client) TopkList(key string) ([]string, error) { 295 | conn := client.Pool.Get() 296 | defer conn.Close() 297 | result, err := conn.Do("TOPK.LIST", key) 298 | return redis.Strings(result, err) 299 | } 300 | 301 | // Returns number of required items (k), width, depth and decay values. 302 | func (client *Client) TopkInfo(key string) (map[string]string, error) { 303 | conn := client.Pool.Get() 304 | defer conn.Close() 305 | reply, err := conn.Do("TOPK.INFO", key) 306 | values, err := redis.Values(reply, err) 307 | if err != nil { 308 | return nil, err 309 | } 310 | if len(values)%2 != 0 { 311 | return nil, errors.New("expects even number of values result") 312 | } 313 | 314 | m := make(map[string]string, len(values)/2) 315 | for i := 0; i < len(values); i += 2 { 316 | k := values[i].(string) 317 | switch v := values[i+1].(type) { 318 | case []byte: 319 | m[k] = string(values[i+1].([]byte)) 320 | break 321 | case int64: 322 | m[k] = strconv.FormatInt(values[i+1].(int64), 10) 323 | default: 324 | return nil, fmt.Errorf("unexpected element type for (Ints,String), got type %T", v) 325 | } 326 | } 327 | return m, err 328 | } 329 | 330 | // Increase the score of an item in the data structure by increment. 331 | func (client *Client) TopkIncrBy(key string, itemIncrements map[string]int64) ([]string, error) { 332 | conn := client.Pool.Get() 333 | defer conn.Close() 334 | args := redis.Args{key} 335 | for k, v := range itemIncrements { 336 | args = args.Add(k, v) 337 | } 338 | reply, err := conn.Do("TOPK.INCRBY", args...) 339 | return redis.Strings(reply, err) 340 | } 341 | 342 | // Initializes a Count-Min Sketch to dimensions specified by user. 343 | func (client *Client) CmsInitByDim(key string, width int64, depth int64) (string, error) { 344 | conn := client.Pool.Get() 345 | defer conn.Close() 346 | result, err := conn.Do("CMS.INITBYDIM", key, width, depth) 347 | return redis.String(result, err) 348 | } 349 | 350 | // Initializes a Count-Min Sketch to accommodate requested capacity. 351 | func (client *Client) CmsInitByProb(key string, error float64, probability float64) (string, error) { 352 | conn := client.Pool.Get() 353 | defer conn.Close() 354 | result, err := conn.Do("CMS.INITBYPROB", key, error, probability) 355 | return redis.String(result, err) 356 | } 357 | 358 | // Increases the count of item by increment. Multiple items can be increased with one call. 359 | func (client *Client) CmsIncrBy(key string, itemIncrements map[string]int64) ([]int64, error) { 360 | conn := client.Pool.Get() 361 | defer conn.Close() 362 | args := redis.Args{key} 363 | for k, v := range itemIncrements { 364 | args = args.Add(k, v) 365 | } 366 | result, err := conn.Do("CMS.INCRBY", args...) 367 | return redis.Int64s(result, err) 368 | } 369 | 370 | // Returns count for item. 371 | func (client *Client) CmsQuery(key string, items []string) ([]int64, error) { 372 | conn := client.Pool.Get() 373 | defer conn.Close() 374 | args := redis.Args{key}.AddFlat(items) 375 | result, err := conn.Do("CMS.QUERY", args...) 376 | return redis.Int64s(result, err) 377 | } 378 | 379 | // Merges several sketches into one sketch, stored at dest key 380 | // All sketches must have identical width and depth. 381 | func (client *Client) CmsMerge(dest string, srcs []string, weights []int64) (string, error) { 382 | conn := client.Pool.Get() 383 | defer conn.Close() 384 | args := redis.Args{dest}.Add(len(srcs)).AddFlat(srcs) 385 | if weights != nil && len(weights) > 0 { 386 | args = args.Add("WEIGHTS").AddFlat(weights) 387 | } 388 | return redis.String(conn.Do("CMS.MERGE", args...)) 389 | } 390 | 391 | // Returns width, depth and total count of the sketch. 392 | func (client *Client) CmsInfo(key string) (map[string]int64, error) { 393 | conn := client.Pool.Get() 394 | defer conn.Close() 395 | return ParseInfoReply(redis.Values(conn.Do("CMS.INFO", key))) 396 | } 397 | 398 | // Create an empty cuckoo filter with an initial capacity of {capacity} items. 399 | func (client *Client) CfReserve(key string, capacity int64, bucketSize int64, maxIterations int64, expansion int64) (string, error) { 400 | conn := client.Pool.Get() 401 | defer conn.Close() 402 | args := redis.Args{key}.Add(capacity) 403 | if bucketSize > 0 { 404 | args = args.Add("BUCKETSIZE", bucketSize) 405 | } 406 | if maxIterations > 0 { 407 | args = args.Add("MAXITERATIONS", maxIterations) 408 | } 409 | if expansion > 0 { 410 | args = args.Add("EXPANSION", expansion) 411 | } 412 | return redis.String(conn.Do("CF.RESERVE", args...)) 413 | } 414 | 415 | // Adds an item to the cuckoo filter, creating the filter if it does not exist. 416 | func (client *Client) CfAdd(key string, item string) (bool, error) { 417 | conn := client.Pool.Get() 418 | defer conn.Close() 419 | return redis.Bool(conn.Do("CF.ADD", key, item)) 420 | } 421 | 422 | // Adds an item to a cuckoo filter if the item did not exist previously. 423 | func (client *Client) CfAddNx(key string, item string) (bool, error) { 424 | conn := client.Pool.Get() 425 | defer conn.Close() 426 | return redis.Bool(conn.Do("CF.ADDNX", key, item)) 427 | } 428 | 429 | // Adds one or more items to a cuckoo filter, allowing the filter to be created with a custom capacity if it does not yet exist. 430 | func (client *Client) CfInsert(key string, cap int64, noCreate bool, items []string) ([]int64, error) { 431 | conn := client.Pool.Get() 432 | defer conn.Close() 433 | args := GetInsertArgs(key, cap, noCreate, items) 434 | return redis.Int64s(conn.Do("CF.INSERT", args...)) 435 | } 436 | 437 | // Adds one or more items to a cuckoo filter, allowing the filter to be created with a custom capacity if it does not yet exist. 438 | func (client *Client) CfInsertNx(key string, cap int64, noCreate bool, items []string) ([]int64, error) { 439 | conn := client.Pool.Get() 440 | defer conn.Close() 441 | args := GetInsertArgs(key, cap, noCreate, items) 442 | return redis.Int64s(conn.Do("CF.INSERTNX", args...)) 443 | } 444 | 445 | func GetInsertArgs(key string, cap int64, noCreate bool, items []string) redis.Args { 446 | args := redis.Args{key} 447 | if cap > 0 { 448 | args = args.Add("CAPACITY", cap) 449 | } 450 | if noCreate { 451 | args = args.Add("NOCREATE") 452 | } 453 | args = args.Add("ITEMS").AddFlat(items) 454 | return args 455 | } 456 | 457 | // Check if an item exists in a Cuckoo Filter 458 | func (client *Client) CfExists(key string, item string) (bool, error) { 459 | conn := client.Pool.Get() 460 | defer conn.Close() 461 | return redis.Bool(conn.Do("CF.EXISTS", key, item)) 462 | } 463 | 464 | // Deletes an item once from the filter. 465 | func (client *Client) CfDel(key string, item string) (bool, error) { 466 | conn := client.Pool.Get() 467 | defer conn.Close() 468 | return redis.Bool(conn.Do("CF.DEL", key, item)) 469 | } 470 | 471 | // Returns the number of times an item may be in the filter. 472 | func (client *Client) CfCount(key string, item string) (int64, error) { 473 | conn := client.Pool.Get() 474 | defer conn.Close() 475 | return redis.Int64(conn.Do("CF.COUNT", key, item)) 476 | } 477 | 478 | // Begins an incremental save of the cuckoo filter. 479 | func (client *Client) CfScanDump(key string, iter int64) (int64, []byte, error) { 480 | conn := client.Pool.Get() 481 | defer conn.Close() 482 | reply, err := redis.Values(conn.Do("CF.SCANDUMP", key, iter)) 483 | if err != nil || len(reply) != 2 { 484 | return 0, nil, err 485 | } 486 | iter = reply[0].(int64) 487 | if reply[1] == nil { 488 | return iter, nil, err 489 | } 490 | return iter, reply[1].([]byte), err 491 | } 492 | 493 | // Restores a filter previously saved using SCANDUMP 494 | func (client *Client) CfLoadChunk(key string, iter int64, data []byte) (string, error) { 495 | conn := client.Pool.Get() 496 | defer conn.Close() 497 | return redis.String(conn.Do("CF.LOADCHUNK", key, iter, data)) 498 | } 499 | 500 | // Return information about key 501 | func (client *Client) CfInfo(key string) (map[string]int64, error) { 502 | conn := client.Pool.Get() 503 | defer conn.Close() 504 | return ParseInfoReply(redis.Values(conn.Do("CF.INFO", key))) 505 | } 506 | 507 | // TdCreate - Allocate the memory and initialize the t-digest 508 | func (client *Client) TdCreate(key string, compression int64) (string, error) { 509 | conn := client.Pool.Get() 510 | defer conn.Close() 511 | return redis.String(conn.Do("TDIGEST.CREATE", key, "COMPRESSION", compression)) 512 | } 513 | 514 | // TdReset - Reset the sketch to zero - empty out the sketch and re-initialize it 515 | func (client *Client) TdReset(key string) (string, error) { 516 | conn := client.Pool.Get() 517 | defer conn.Close() 518 | return redis.String(conn.Do("TDIGEST.RESET", key)) 519 | } 520 | 521 | // TdAdd - Adds one or more samples to a sketch 522 | func (client *Client) TdAdd(key string, samples map[float64]float64) (string, error) { 523 | conn := client.Pool.Get() 524 | defer conn.Close() 525 | args := redis.Args{key} 526 | for k, v := range samples { 527 | args = args.Add(k, v) 528 | } 529 | reply, err := conn.Do("TDIGEST.ADD", args...) 530 | return redis.String(reply, err) 531 | } 532 | 533 | // tdMerge - The internal representation of TdMerge. All underlying functions call this one, 534 | // returning its results. It allows us to maintain interfaces. 535 | // see https://redis.io/commands/tdigest.merge/ 536 | // 537 | // The default values for compression is 100 538 | func (client *Client) tdMerge(toKey string, compression int64, override bool, numKeys int64, fromKey ...string) (string, error) { 539 | if numKeys < 1 { 540 | return "", errors.New("a minimum of one key must be merged") 541 | } 542 | 543 | conn := client.Pool.Get() 544 | defer conn.Close() 545 | overidable := "" 546 | if override { 547 | overidable = "1" 548 | } 549 | return redis.String(conn.Do("TDIGEST.MERGE", toKey, 550 | strconv.FormatInt(numKeys, 10), 551 | strings.Join(fromKey, " "), 552 | "COMPRESSION", compression, 553 | overidable)) 554 | } 555 | 556 | // TdMerge - Merges all of the values from 'from' to 'this' sketch 557 | func (client *Client) TdMerge(toKey string, numKeys int64, fromKey ...string) (string, error) { 558 | return client.tdMerge(toKey, 100, false, numKeys, fromKey...) 559 | } 560 | 561 | // TdMergeWithCompression - Merges all of the values from 'from' to 'this' sketch with specified compression 562 | func (client *Client) TdMergeWithCompression(toKey string, compression int64, numKeys int64, fromKey ...string) (string, error) { 563 | return client.tdMerge(toKey, compression, false, numKeys, fromKey...) 564 | } 565 | 566 | // TdMergeWithOverride - Merges all of the values from 'from' to 'this' sketch overriding the destination key if it exists 567 | func (client *Client) TdMergeWithOverride(toKey string, override bool, numKeys int64, fromKey ...string) (string, error) { 568 | return client.tdMerge(toKey, 100, true, numKeys, fromKey...) 569 | } 570 | 571 | // TdMergeWithCompressionAndOverride - Merges all of the values from 'from' to 'this' sketch with specified compression 572 | // and overriding the destination key if it exists 573 | func (client *Client) TdMergeWithCompressionAndOverride(toKey string, compression int64, numKeys int64, fromKey ...string) (string, error) { 574 | return client.tdMerge(toKey, compression, true, numKeys, fromKey...) 575 | } 576 | 577 | // TdMin - Get minimum value from the sketch. Will return DBL_MAX if the sketch is empty 578 | func (client *Client) TdMin(key string) (float64, error) { 579 | conn := client.Pool.Get() 580 | defer conn.Close() 581 | return redis.Float64(conn.Do("TDIGEST.MIN", key)) 582 | } 583 | 584 | // TdMax - Get maximum value from the sketch. Will return DBL_MIN if the sketch is empty 585 | func (client *Client) TdMax(key string) (float64, error) { 586 | conn := client.Pool.Get() 587 | defer conn.Close() 588 | return redis.Float64(conn.Do("TDIGEST.MAX", key)) 589 | } 590 | 591 | // TdQuantile - Returns an estimate of the cutoff such that a specified fraction of the data added 592 | // to this TDigest would be less than or equal to the cutoff 593 | func (client *Client) TdQuantile(key string, quantile float64) ([]float64, error) { 594 | conn := client.Pool.Get() 595 | defer conn.Close() 596 | return redis.Float64s(conn.Do("TDIGEST.QUANTILE", key, quantile)) 597 | } 598 | 599 | // TdCdf - Returns the list of fractions of all points added which are <= values 600 | func (client *Client) TdCdf(key string, values ...float64) ([]float64, error) { 601 | conn := client.Pool.Get() 602 | defer conn.Close() 603 | 604 | args := make([]string, len(values)) 605 | for idx, obj := range values { 606 | args[idx] = strconv.FormatFloat(obj, 'f', -1, 64) 607 | } 608 | return redis.Float64s(conn.Do("TDIGEST.CDF", key, strings.Join(args, " "))) 609 | } 610 | 611 | // TdInfo - Returns compression, capacity, total merged and unmerged nodes, the total 612 | // compressions made up to date on that key, and merged and unmerged weight. 613 | func (client *Client) TdInfo(key string) (TDigestInfo, error) { 614 | conn := client.Pool.Get() 615 | defer conn.Close() 616 | return ParseTDigestInfo(redis.Values(conn.Do("TDIGEST.INFO", key))) 617 | } 618 | 619 | func ParseInfoReply(values []interface{}, err error) (map[string]int64, error) { 620 | if err != nil { 621 | return nil, err 622 | } 623 | if len(values)%2 != 0 { 624 | return nil, errors.New("expects even number of values result") 625 | } 626 | m := make(map[string]int64, len(values)/2) 627 | for i := 0; i < len(values); i += 2 { 628 | m[values[i].(string)] = values[i+1].(int64) 629 | } 630 | return m, err 631 | } 632 | 633 | func ParseTDigestInfo(result interface{}, err error) (info TDigestInfo, outErr error) { 634 | values, outErr := redis.Values(result, err) 635 | if outErr != nil { 636 | return TDigestInfo{}, err 637 | } 638 | if len(values)%2 != 0 { 639 | return TDigestInfo{}, errors.New("ParseInfo expects even number of values result") 640 | } 641 | var key string 642 | for i := 0; i < len(values); i += 2 { 643 | key, outErr = redis.String(values[i], nil) 644 | if outErr != nil { 645 | return TDigestInfo{}, outErr 646 | } 647 | switch key { 648 | case "Compression": 649 | info.compression, outErr = redis.Int64(values[i+1], nil) 650 | case "Capacity": 651 | info.capacity, outErr = redis.Int64(values[i+1], nil) 652 | case "Merged nodes": 653 | info.mergedNodes, outErr = redis.Int64(values[i+1], nil) 654 | case "Unmerged nodes": 655 | info.unmergedNodes, outErr = redis.Int64(values[i+1], nil) 656 | case "Merged weight": 657 | info.mergedWeight, outErr = redis.Int64(values[i+1], nil) 658 | case "Unmerged weight": 659 | info.unmergedWeight, outErr = redis.Int64(values[i+1], nil) 660 | case "Total compressions": 661 | info.totalCompressions, outErr = redis.Int64(values[i+1], nil) 662 | } 663 | if outErr != nil { 664 | return TDigestInfo{}, outErr 665 | } 666 | } 667 | 668 | return info, nil 669 | } 670 | -------------------------------------------------------------------------------- /client_test.go: -------------------------------------------------------------------------------- 1 | package redis_bloom_go 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "time" 7 | 8 | "github.com/gomodule/redigo/redis" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func getTestConnectionDetails() (string, string) { 13 | value, exists := os.LookupEnv("REDISBLOOM_TEST_HOST") 14 | host := "localhost:6379" 15 | password := "" 16 | valuePassword, existsPassword := os.LookupEnv("REDISBLOOM_TEST_PASSWORD") 17 | if exists && value != "" { 18 | host = value 19 | } 20 | if existsPassword && valuePassword != "" { 21 | password = valuePassword 22 | } 23 | return host, password 24 | } 25 | 26 | func createClient() *Client { 27 | host, password := getTestConnectionDetails() 28 | var ptr *string = nil 29 | if len(password) > 0 { 30 | ptr = &password 31 | } 32 | return NewClient(host, "test_client", ptr) 33 | } 34 | 35 | func TestNewClientFromPool(t *testing.T) { 36 | host, password := getTestConnectionDetails() 37 | pool := &redis.Pool{Dial: func() (redis.Conn, error) { 38 | return redis.Dial("tcp", host, redis.DialPassword(password)) 39 | }, MaxIdle: maxConns} 40 | client1 := NewClientFromPool(pool, "bloom-client-1") 41 | client2 := NewClientFromPool(pool, "bloom-client-2") 42 | assert.Equal(t, client1.Pool, client2.Pool) 43 | err1 := client1.Pool.Close() 44 | err2 := client2.Pool.Close() 45 | assert.Nil(t, err1) 46 | assert.Nil(t, err2) 47 | } 48 | 49 | var client = createClient() 50 | var _ = client.FlushAll() 51 | 52 | var defaultDuration, _ = time.ParseDuration("1h") 53 | var tooShortDuration, _ = time.ParseDuration("10ms") 54 | 55 | func (client *Client) FlushAll() (err error) { 56 | conn := client.Pool.Get() 57 | defer conn.Close() 58 | _, err = conn.Do("FLUSHALL") 59 | return err 60 | } 61 | 62 | func TestReserve(t *testing.T) { 63 | client.FlushAll() 64 | key := "test_RESERVE" 65 | err := client.Reserve(key, 0.1, 1000) 66 | assert.Nil(t, err) 67 | 68 | info, err := client.Info(key) 69 | assert.Nil(t, err) 70 | assert.Equal(t, info, map[string]int64{ 71 | "Capacity": 1000, 72 | "Expansion rate": 2, 73 | "Number of filters": 1, 74 | "Number of items inserted": 0, 75 | "Size": 880, 76 | }) 77 | 78 | err = client.Reserve(key, 0.1, 1000) 79 | assert.NotNil(t, err) 80 | } 81 | 82 | func TestAdd(t *testing.T) { 83 | client.FlushAll() 84 | key := "test_ADD" 85 | value := "test_ADD_value" 86 | exists, err := client.Add(key, value) 87 | assert.Nil(t, err) 88 | assert.True(t, exists) 89 | 90 | info, err := client.Info(key) 91 | assert.Nil(t, err) 92 | assert.NotNil(t, info) 93 | 94 | exists, err = client.Add(key, value) 95 | assert.Nil(t, err) 96 | assert.False(t, exists) 97 | } 98 | 99 | func TestExists(t *testing.T) { 100 | client.FlushAll() 101 | client.Add("test_ADD", "test_EXISTS") 102 | 103 | exists, err := client.Exists("test_ADD", "test_EXISTS") 104 | assert.Nil(t, err) 105 | assert.True(t, exists) 106 | 107 | exists, err = client.Exists("test_ADD", "test_EXISTS1") 108 | assert.Nil(t, err) 109 | assert.False(t, exists) 110 | } 111 | 112 | func TestClient_BfAddMulti(t *testing.T) { 113 | client.FlushAll() 114 | ret, err := client.BfAddMulti("test_add_multi", []string{"a", "b", "c"}) 115 | assert.Nil(t, err) 116 | assert.NotNil(t, ret) 117 | } 118 | 119 | func TestClient_BfCard(t *testing.T) { 120 | keyname := "bfcardkey" 121 | client.FlushAll() 122 | res, err := client.BfCard(keyname) 123 | assert.Nil(t, err) 124 | assert.Equal(t, res, int64(0)) 125 | 126 | client.FlushAll() 127 | client.BfAddMulti(keyname, []string{"a", "b", "c"}) 128 | res, err = client.BfCard(keyname) 129 | assert.Nil(t, err) 130 | assert.Equal(t, res, int64(3)) 131 | } 132 | 133 | func TestClient_BfExistsMulti(t *testing.T) { 134 | client.FlushAll() 135 | key := "test_exists_multi" 136 | ret, err := client.BfAddMulti(key, []string{"a", "b", "c"}) 137 | assert.Nil(t, err) 138 | assert.NotNil(t, ret) 139 | 140 | existsResult, err := client.BfExistsMulti(key, []string{"a", "b", "notexists"}) 141 | assert.Nil(t, err) 142 | assert.Equal(t, 3, len(existsResult)) 143 | assert.Equal(t, int64(1), existsResult[0]) 144 | assert.Equal(t, int64(1), existsResult[1]) 145 | assert.Equal(t, int64(0), existsResult[2]) 146 | } 147 | 148 | func TestClient_BfInsert(t *testing.T) { 149 | client.FlushAll() 150 | key := "test_bf_insert" 151 | key_expansion := "test_bf_insert_expansion" 152 | key_nocreate := "test_bf_insert_nocreate" 153 | key_noscaling := "test_bf_insert_noscaling" 154 | 155 | ret, err := client.BfInsert(key, 1000, 0.1, -1, false, false, []string{"a"}) 156 | assert.Nil(t, err) 157 | assert.Equal(t, 1, len(ret)) 158 | assert.True(t, ret[0] > 0) 159 | existsResult, err := client.BfExistsMulti(key, []string{"a"}) 160 | assert.Nil(t, err) 161 | assert.Equal(t, 1, len(existsResult)) 162 | assert.Equal(t, int64(1), existsResult[0]) 163 | 164 | ret, err = client.BfInsert(key, 1000, 0.1, -1, false, false, []string{"a", "b"}) 165 | assert.Nil(t, err) 166 | assert.Equal(t, 2, len(ret)) 167 | 168 | // Test for EXPANSION : If a new sub-filter is created, its size will be the size of the current filter multiplied by expansion 169 | ret, err = client.BfInsert(key_expansion, 1000, 0.1, 4, false, false, []string{"a"}) 170 | assert.Nil(t, err) 171 | assert.Equal(t, 1, len(ret)) 172 | 173 | // Test for NOCREATE : If specified, indicates that the filter should not be created if it does not already exist 174 | _, err = client.BfInsert(key_nocreate, 1000, 0.1, -1, true, false, []string{"a"}) 175 | assert.NotNil(t, err) 176 | 177 | // Test NONSCALING : Prevents the filter from creating additional sub-filters if initial capacity is reached. 178 | ret, err = client.BfInsert(key_noscaling, 2, 0.1, -1, false, true, []string{"a", "b"}) 179 | assert.Nil(t, err) 180 | assert.Equal(t, 2, len(ret)) 181 | ret, err = client.BfInsert(key_noscaling, 2, 0.1, -1, false, true, []string{"c"}) 182 | assert.NotNil(t, err) 183 | assert.Equal(t, 0, len(ret)) 184 | assert.Equal(t, err.Error(), "ERR non scaling filter is full") 185 | } 186 | 187 | func TestClient_TopkReserve(t *testing.T) { 188 | client.FlushAll() 189 | ret, err := client.TopkReserve("test_topk_reserve", 10, 2000, 7, 0.925) 190 | assert.Nil(t, err) 191 | assert.Equal(t, "OK", ret) 192 | } 193 | 194 | func TestClient_TopkAdd(t *testing.T) { 195 | client.FlushAll() 196 | key := "test_topk_add" 197 | ret, err := client.TopkReserve(key, 10, 2000, 7, 0.925) 198 | assert.Nil(t, err) 199 | assert.Equal(t, "OK", ret) 200 | rets, err := client.TopkAdd(key, []string{"test", "test1", "test3"}) 201 | assert.Nil(t, err) 202 | assert.Equal(t, 3, len(rets)) 203 | } 204 | 205 | func TestClient_TopkCount(t *testing.T) { 206 | client.FlushAll() 207 | key := "test_topk_count" 208 | ret, err := client.TopkReserve(key, 10, 2000, 7, 0.925) 209 | assert.Nil(t, err) 210 | assert.Equal(t, "OK", ret) 211 | rets, err := client.TopkAdd(key, []string{"test", "test1", "test3"}) 212 | assert.Nil(t, err) 213 | assert.Equal(t, 3, len(rets)) 214 | counts, err := client.TopkCount(key, []string{"test", "test1", "test3"}) 215 | assert.Equal(t, 3, len(counts)) 216 | for _, element := range counts { 217 | assert.LessOrEqual(t, int64(1), element) 218 | } 219 | } 220 | 221 | func TestClient_TopkQuery(t *testing.T) { 222 | client.FlushAll() 223 | key := "test_topk_query" 224 | ret, err := client.TopkReserve(key, 10, 2000, 7, 0.925) 225 | assert.Nil(t, err) 226 | assert.Equal(t, "OK", ret) 227 | rets, err := client.TopkAdd(key, []string{"test"}) 228 | assert.Nil(t, err) 229 | assert.NotNil(t, rets) 230 | queryRet, err := client.TopkQuery(key, []string{"test", "nonexist"}) 231 | assert.Nil(t, err) 232 | assert.Equal(t, 2, len(queryRet)) 233 | assert.Equal(t, int64(1), queryRet[0]) 234 | assert.Equal(t, int64(0), queryRet[1]) 235 | 236 | key1 := "test_topk_list" 237 | ret, err = client.TopkReserve(key1, 3, 50, 3, 0.9) 238 | assert.Nil(t, err) 239 | assert.Equal(t, "OK", ret) 240 | client.TopkAdd(key1, []string{"A", "B", "C", "D", "E", "A", "A", "B", "C", 241 | "G", "D", "B", "D", "A", "E", "E"}) 242 | keys, err := client.TopkList(key1) 243 | assert.Nil(t, err) 244 | assert.Equal(t, 3, len(keys)) 245 | assert.Equal(t, []string{"A", "B", "E"}, keys) 246 | 247 | // WithCount option 248 | keysWithCount, err := client.TopkListWithCount(key1) 249 | assert.Nil(t, err) 250 | assert.Equal(t, 3, len(keysWithCount)) 251 | assert.Equal(t, map[string]int64{"A": 4, "B": 3, "E": 3}, keysWithCount) 252 | } 253 | 254 | func TestClient_TopkInfo(t *testing.T) { 255 | client.FlushAll() 256 | key := "test_topk_info" 257 | ret, err := client.TopkReserve(key, 10, 2000, 7, 0.925) 258 | assert.Nil(t, err) 259 | assert.Equal(t, "OK", ret) 260 | 261 | info, err := client.TopkInfo(key) 262 | assert.Equal(t, "10", info["k"]) 263 | assert.Equal(t, "2000", info["width"]) 264 | assert.Equal(t, "7", info["depth"]) 265 | 266 | info, err = client.TopkInfo("notexists") 267 | assert.NotNil(t, err) 268 | } 269 | 270 | func TestClient_TopkIncrBy(t *testing.T) { 271 | client.FlushAll() 272 | key := "test_topk_incrby" 273 | ret, err := client.TopkReserve(key, 50, 2000, 7, 0.925) 274 | assert.Nil(t, err) 275 | assert.Equal(t, "OK", ret) 276 | 277 | rets, err := client.TopkAdd(key, []string{"foo", "bar", "42"}) 278 | assert.Nil(t, err) 279 | assert.NotNil(t, rets) 280 | 281 | rets, err = client.TopkIncrBy(key, map[string]int64{"foo": 3, "bar": 2, "42": 30}) 282 | assert.Nil(t, err) 283 | assert.Equal(t, 3, len(rets)) 284 | assert.Equal(t, "", rets[2]) 285 | } 286 | 287 | func TestClient_CmsInitByDim(t *testing.T) { 288 | client.FlushAll() 289 | ret, err := client.CmsInitByDim("test_cms_initbydim", 1000, 5) 290 | assert.Nil(t, err) 291 | assert.Equal(t, "OK", ret) 292 | } 293 | 294 | func TestClient_CmsInitByProb(t *testing.T) { 295 | client.FlushAll() 296 | ret, err := client.CmsInitByProb("test_cms_initbyprob", 0.01, 0.01) 297 | assert.Nil(t, err) 298 | assert.Equal(t, "OK", ret) 299 | } 300 | 301 | func TestClient_CmsIncrBy(t *testing.T) { 302 | client.FlushAll() 303 | key := "test_cms_incrby" 304 | ret, err := client.CmsInitByDim(key, 1000, 5) 305 | assert.Nil(t, err) 306 | assert.Equal(t, "OK", ret) 307 | results, err := client.CmsIncrBy(key, map[string]int64{"foo": 5}) 308 | assert.Nil(t, err) 309 | assert.NotNil(t, results) 310 | assert.Equal(t, int64(5), results[0]) 311 | } 312 | 313 | func TestClient_CmsQuery(t *testing.T) { 314 | client.FlushAll() 315 | key := "test_cms_query" 316 | ret, err := client.CmsInitByDim(key, 1000, 5) 317 | assert.Nil(t, err) 318 | assert.Equal(t, "OK", ret) 319 | results, err := client.CmsQuery(key, []string{"notexist"}) 320 | assert.Nil(t, err) 321 | assert.NotNil(t, 0, results[0]) 322 | _, err = client.CmsIncrBy(key, map[string]int64{"foo": 5}) 323 | assert.Nil(t, err) 324 | results, err = client.CmsQuery(key, []string{"foo"}) 325 | assert.Nil(t, err) 326 | assert.Equal(t, int64(5), results[0]) 327 | } 328 | 329 | func TestClient_CmsMerge(t *testing.T) { 330 | client.FlushAll() 331 | ret, err := client.CmsInitByDim("A", 1000, 5) 332 | assert.Nil(t, err) 333 | assert.Equal(t, "OK", ret) 334 | ret, err = client.CmsInitByDim("B", 1000, 5) 335 | assert.Nil(t, err) 336 | assert.Equal(t, "OK", ret) 337 | ret, err = client.CmsInitByDim("C", 1000, 5) 338 | assert.Nil(t, err) 339 | assert.Equal(t, "OK", ret) 340 | ret, err = client.CmsInitByDim("D", 1000, 5) 341 | assert.Nil(t, err) 342 | assert.Equal(t, "OK", ret) 343 | ret, err = client.CmsInitByDim("E", 1000, 5) 344 | assert.Nil(t, err) 345 | assert.Equal(t, "OK", ret) 346 | 347 | client.CmsIncrBy("A", map[string]int64{"foo": 5, "bar": 3, "baz": 9}) 348 | client.CmsIncrBy("B", map[string]int64{"foo": 2, "bar": 3, "baz": 1}) 349 | 350 | // Negative test ( key not exist ) 351 | ret, err = client.CmsMerge("dont_exist", []string{"A", "B"}, nil) 352 | assert.NotNil(t, err) 353 | assert.Equal(t, "CMS: key does not exist", err.Error()) 354 | 355 | // Positive tests 356 | ret, err = client.CmsMerge("C", []string{"A", "B"}, nil) 357 | assert.Nil(t, err) 358 | assert.Equal(t, "OK", ret) 359 | results, err := client.CmsQuery("C", []string{"foo", "bar", "baz"}) 360 | assert.Equal(t, []int64{7, 6, 10}, results) 361 | 362 | // Test for WEIGHTS ( default weight ) 363 | ret, err = client.CmsMerge("D", []string{"A", "B"}, []int64{1, 1, 1}) 364 | assert.Nil(t, err) 365 | assert.Equal(t, "OK", ret) 366 | results, err = client.CmsQuery("D", []string{"foo", "bar", "baz"}) 367 | assert.Equal(t, []int64{7, 6, 10}, results) 368 | 369 | // Test for WEIGHTS ( default weight ) 370 | ret, err = client.CmsMerge("E", []string{"A", "B"}, []int64{1, 5}) 371 | assert.Nil(t, err) 372 | assert.Equal(t, "OK", ret) 373 | results, err = client.CmsQuery("E", []string{"foo", "bar", "baz"}) 374 | assert.Equal(t, []int64{5 + 2*5, 3 + 3*5, 9 + 1*5}, results) 375 | } 376 | 377 | func TestClient_CmsInfo(t *testing.T) { 378 | client.FlushAll() 379 | key := "test_cms_info" 380 | ret, err := client.CmsInitByDim(key, 1000, 5) 381 | assert.Nil(t, err) 382 | assert.Equal(t, "OK", ret) 383 | info, err := client.CmsInfo(key) 384 | assert.Nil(t, err) 385 | assert.Equal(t, int64(1000), info["width"]) 386 | assert.Equal(t, int64(5), info["depth"]) 387 | assert.Equal(t, int64(0), info["count"]) 388 | } 389 | 390 | func TestClient_CfReserve(t *testing.T) { 391 | client.FlushAll() 392 | key := "test_cf_reserve" 393 | key_max_iterations := "test_cf_reserve_maxiterations" 394 | key_expansion := "test_cf_reserve_expansion" 395 | ret, err := client.CfReserve(key, 1000, -1, -1, -1) 396 | assert.Nil(t, err) 397 | assert.Equal(t, "OK", ret) 398 | 399 | // Test for MAXITERATIONS property 400 | ret, err = client.CfReserve(key_max_iterations, 1000, -1, 20, -1) 401 | assert.Nil(t, err) 402 | assert.Equal(t, "OK", ret) 403 | 404 | // Test for EXPANSION property 405 | ret, err = client.CfReserve(key_expansion, 1000, -1, -1, 2) 406 | assert.Nil(t, err) 407 | assert.Equal(t, "OK", ret) 408 | } 409 | 410 | func TestClient_CfAdd(t *testing.T) { 411 | client.FlushAll() 412 | key := "test_cf_add" 413 | ret, err := client.CfAdd(key, "a") 414 | assert.Nil(t, err) 415 | assert.True(t, ret) 416 | ret, err = client.CfAddNx(key, "b") 417 | assert.Nil(t, err) 418 | assert.True(t, ret) 419 | } 420 | 421 | func TestClient_CfInsert(t *testing.T) { 422 | client.FlushAll() 423 | key := "test_cf_insert" 424 | ret, err := client.CfInsert(key, 1000, false, []string{"a"}) 425 | assert.Nil(t, err) 426 | assert.Equal(t, 1, len(ret)) 427 | assert.True(t, ret[0] > 0) 428 | ret, err = client.CfInsertNx(key, 1000, true, []string{"b"}) 429 | assert.Nil(t, err) 430 | assert.Equal(t, 1, len(ret)) 431 | assert.True(t, ret[0] > 0) 432 | } 433 | 434 | func TestClient_CfExists(t *testing.T) { 435 | client.FlushAll() 436 | key := "test_cf_exists" 437 | ret, err := client.CfAdd(key, "a") 438 | assert.Nil(t, err) 439 | assert.True(t, ret) 440 | ret, err = client.CfExists(key, "a") 441 | assert.Nil(t, err) 442 | assert.True(t, ret) 443 | } 444 | 445 | func TestClient_CfDel(t *testing.T) { 446 | client.FlushAll() 447 | key := "test_cf_del" 448 | ret, err := client.CfAdd(key, "a") 449 | assert.Nil(t, err) 450 | assert.True(t, ret) 451 | ret, err = client.CfExists(key, "a") 452 | assert.Nil(t, err) 453 | assert.True(t, ret) 454 | ret, err = client.CfDel(key, "a") 455 | assert.Nil(t, err) 456 | assert.True(t, ret) 457 | ret, err = client.CfExists(key, "a") 458 | assert.Nil(t, err) 459 | assert.False(t, ret) 460 | } 461 | 462 | func TestClient_CfCount(t *testing.T) { 463 | client.FlushAll() 464 | key := "test_cf_count" 465 | ret, err := client.CfAdd(key, "a") 466 | assert.Nil(t, err) 467 | assert.True(t, ret) 468 | count, err := client.CfCount(key, "a") 469 | assert.Nil(t, err) 470 | assert.Equal(t, int64(1), count) 471 | } 472 | 473 | func TestClient_CfScanDump(t *testing.T) { 474 | client.FlushAll() 475 | key := "test_cf_scandump" 476 | ret, err := client.CfReserve(key, 100, 50, -1, -1) 477 | assert.Nil(t, err) 478 | assert.Equal(t, "OK", ret) 479 | client.CfAdd(key, "a") 480 | curIter := int64(0) 481 | chunks := make([]map[string]interface{}, 0) 482 | for { 483 | iter, data, err := client.CfScanDump(key, curIter) 484 | assert.Nil(t, err) 485 | curIter = iter 486 | if iter == int64(0) { 487 | break 488 | } 489 | chunk := map[string]interface{}{"iter": iter, "data": data} 490 | chunks = append(chunks, chunk) 491 | } 492 | client.FlushAll() 493 | for i := 0; i < len(chunks); i++ { 494 | ret, err := client.CfLoadChunk(key, chunks[i]["iter"].(int64), chunks[i]["data"].([]byte)) 495 | assert.Nil(t, err) 496 | assert.Equal(t, "OK", ret) 497 | } 498 | exists, err := client.CfExists(key, "a") 499 | assert.True(t, exists) 500 | } 501 | 502 | func TestClient_CfInfo(t *testing.T) { 503 | client.FlushAll() 504 | key := "test_cf_info" 505 | ret, err := client.CfAdd(key, "a") 506 | assert.Nil(t, err) 507 | assert.True(t, ret) 508 | info, err := client.CfInfo(key) 509 | assert.Equal(t, int64(1080), info["Size"]) 510 | assert.Equal(t, int64(512), info["Number of buckets"]) 511 | assert.Equal(t, int64(0), info["Number of filter"]) 512 | assert.Equal(t, int64(1), info["Number of items inserted"]) 513 | assert.Equal(t, int64(0), info["Max iteration"]) 514 | } 515 | 516 | func TestClient_BfScanDump(t *testing.T) { 517 | client.FlushAll() 518 | key := "test_bf_scandump" 519 | err := client.Reserve(key, 0.01, 1000) 520 | assert.Nil(t, err) 521 | client.Add(key, "1") 522 | curIter := int64(0) 523 | chunks := make([]map[string]interface{}, 0) 524 | for { 525 | iter, data, err := client.BfScanDump(key, curIter) 526 | assert.Nil(t, err) 527 | curIter = iter 528 | if iter == int64(0) { 529 | break 530 | } 531 | chunk := map[string]interface{}{"iter": iter, "data": data} 532 | chunks = append(chunks, chunk) 533 | } 534 | client.FlushAll() 535 | for i := 0; i < len(chunks); i++ { 536 | ret, err := client.BfLoadChunk(key, chunks[i]["iter"].(int64), chunks[i]["data"].([]byte)) 537 | assert.Nil(t, err) 538 | assert.Equal(t, "OK", ret) 539 | } 540 | exists, err := client.Exists(key, "1") 541 | assert.True(t, exists) 542 | 543 | // Negative testing 544 | notBfKey := "string_key" 545 | conn := client.Pool.Get() 546 | defer conn.Close() 547 | _, err = conn.Do("SET", redis.Args{notBfKey, "value"}...) 548 | assert.Nil(t, err) 549 | _, _, err = client.BfScanDump(notBfKey, 0) 550 | assert.Equal(t, err.Error(), "WRONGTYPE Operation against a key holding the wrong kind of value") 551 | } 552 | 553 | func TestClient_TdReset(t *testing.T) { 554 | client.FlushAll() 555 | key := "test_td" 556 | ret, err := client.TdCreate(key, 100) 557 | assert.Nil(t, err) 558 | assert.Equal(t, "OK", ret) 559 | 560 | ret, err = client.TdReset(key) 561 | assert.Nil(t, err) 562 | assert.Equal(t, "OK", ret) 563 | 564 | samples := map[float64]float64{1.0: 1.0, 2.0: 2.0} 565 | ret, err = client.TdAdd(key, samples) 566 | assert.Nil(t, err) 567 | assert.Equal(t, "OK", ret) 568 | 569 | ret, err = client.TdReset(key) 570 | assert.Nil(t, err) 571 | assert.Equal(t, "OK", ret) 572 | } 573 | 574 | func TestClient_TdMerge(t *testing.T) { 575 | client.FlushAll() 576 | key1 := "toKey" 577 | key2 := "fromKey" 578 | ret, err := client.TdCreate(key1, 10) 579 | assert.Nil(t, err) 580 | assert.Equal(t, "OK", ret) 581 | ret, err = client.TdCreate(key2, 10) 582 | assert.Nil(t, err) 583 | assert.Equal(t, "OK", ret) 584 | 585 | //Add values 586 | samples1 := map[float64]float64{1.0: 1.0, 2.0: 2.0} 587 | samples2 := map[float64]float64{3.0: 3.0, 4.0: 4.0} 588 | ret, err = client.TdAdd(key1, samples1) 589 | assert.Nil(t, err) 590 | assert.Equal(t, "OK", ret) 591 | ret, err = client.TdAdd(key2, samples2) 592 | assert.Nil(t, err) 593 | assert.Equal(t, "OK", ret) 594 | 595 | //Merge 596 | ret, err = client.TdMerge(key1, 1, key2) 597 | assert.Nil(t, err) 598 | assert.Equal(t, "OK", ret) 599 | 600 | // TODO 601 | // we should now have 10 weight on to-histogram 602 | info, err := client.TdInfo(key1) 603 | assert.Nil(t, err) 604 | assert.Equal(t, int64(8), info.UnmergedWeight()+info.MergedWeight()) 605 | assert.Equal(t, int64(4), info.UnmergedNodes()) 606 | assert.Equal(t, int64(4), info.MergedNodes()) 607 | } 608 | 609 | func TestClient_TdMinMax(t *testing.T) { 610 | client.FlushAll() 611 | key := "test_td" 612 | ret, err := client.TdCreate(key, 10) 613 | assert.Nil(t, err) 614 | assert.Equal(t, "OK", ret) 615 | 616 | samples := map[float64]float64{1.0: 1.0, 2.0: 2.0, 3.0: 3.0} 617 | ret, err = client.TdAdd(key, samples) 618 | assert.Nil(t, err) 619 | assert.Equal(t, "OK", ret) 620 | 621 | ans, err := client.TdMin(key) 622 | assert.Nil(t, err) 623 | assert.Equal(t, 1.0, ans) 624 | 625 | ans, err = client.TdMax(key) 626 | assert.Nil(t, err) 627 | assert.Equal(t, 3.0, ans) 628 | } 629 | 630 | func TestClient_TdQuantile(t *testing.T) { 631 | client.FlushAll() 632 | key := "test_td" 633 | ret, err := client.TdCreate(key, 10) 634 | assert.Nil(t, err) 635 | assert.Equal(t, "OK", ret) 636 | 637 | samples := map[float64]float64{1.0: 1.0, 2.0: 1.0, 3.0: 1.0} 638 | ret, err = client.TdAdd(key, samples) 639 | assert.Nil(t, err) 640 | assert.Equal(t, "OK", ret) 641 | 642 | ans, err := client.TdQuantile(key, 1.0) 643 | assert.Nil(t, err) 644 | assert.Equal(t, 3.0, ans[0]) 645 | 646 | ans, err = client.TdQuantile(key, 0.0) 647 | assert.Nil(t, err) 648 | assert.Equal(t, 1.0, ans[0]) 649 | } 650 | 651 | func TestClient_TdCdf(t *testing.T) { 652 | client.FlushAll() 653 | key := "test_td" 654 | ret, err := client.TdCreate(key, 10) 655 | assert.Nil(t, err) 656 | assert.Equal(t, "OK", ret) 657 | 658 | samples := map[float64]float64{1.0: 1.0, 2.0: 1.0, 3.0: 1.0} 659 | ret, err = client.TdAdd(key, samples) 660 | assert.Nil(t, err) 661 | assert.Equal(t, "OK", ret) 662 | 663 | ans, err := client.TdCdf(key, 10.0) 664 | assert.Nil(t, err) 665 | assert.Equal(t, 1.0, ans[0]) 666 | 667 | ans, err = client.TdCdf(key, 0.0) 668 | assert.Nil(t, err) 669 | assert.Equal(t, 0.0, ans[0]) 670 | } 671 | -------------------------------------------------------------------------------- /example_client_test.go: -------------------------------------------------------------------------------- 1 | package redis_bloom_go_test 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | 7 | redisbloom "github.com/RedisBloom/redisbloom-go" 8 | "github.com/gomodule/redigo/redis" 9 | ) 10 | 11 | // exemplifies the NewClient function 12 | func ExampleNewClient() { 13 | host := "localhost:6379" 14 | var client = redisbloom.NewClient(host, "nohelp", nil) 15 | 16 | // BF.ADD mytest item 17 | _, err := client.Add("mytest", "myItem") 18 | if err != nil { 19 | fmt.Println("Error:", err) 20 | } 21 | 22 | exists, err := client.Exists("mytest", "myItem") 23 | if err != nil { 24 | fmt.Println("Error:", err) 25 | } 26 | fmt.Println("myItem exists in mytest: ", exists) 27 | // Output: myItem exists in mytest: true 28 | 29 | } 30 | 31 | // exemplifies the NewClientFromPool function 32 | func ExampleNewClientFromPool() { 33 | host := "localhost:6379" 34 | password := "" 35 | pool := &redis.Pool{Dial: func() (redis.Conn, error) { 36 | return redis.Dial("tcp", host, redis.DialPassword(password)) 37 | }} 38 | client := redisbloom.NewClientFromPool(pool, "bloom-client-1") 39 | 40 | // BF.ADD mytest item 41 | _, err := client.Add("mytest", "myItem") 42 | if err != nil { 43 | log.Fatalf("Error: %v", err) 44 | } 45 | 46 | exists, err := client.Exists("mytest", "myItem") 47 | if err != nil { 48 | log.Fatalf("Error: %v", err) 49 | } 50 | fmt.Println("myItem exists in mytest: ", exists) 51 | // Output: myItem exists in mytest: true 52 | 53 | } 54 | 55 | // exemplifies the TdCreate function 56 | func ExampleClient_TdCreate() { 57 | host := "localhost:6379" 58 | var client = redisbloom.NewClient(host, "nohelp", nil) 59 | client.FlushAll() 60 | 61 | ret, err := client.TdCreate("key", 100) 62 | if err != nil { 63 | fmt.Println("Error:", err) 64 | } 65 | 66 | fmt.Println(ret) 67 | // Output: OK 68 | 69 | } 70 | 71 | // exemplifies the TdAdd function 72 | func ExampleClient_TdAdd() { 73 | host := "localhost:6379" 74 | var client = redisbloom.NewClient(host, "nohelp", nil) 75 | client.FlushAll() 76 | 77 | key := "example" 78 | ret, err := client.TdCreate(key, 100) 79 | if err != nil { 80 | fmt.Println("Error:", err) 81 | } 82 | 83 | samples := map[float64]float64{1.0: 1.0, 2.0: 2.0} 84 | ret, err = client.TdAdd(key, samples) 85 | if err != nil { 86 | fmt.Println("Error:", err) 87 | } 88 | 89 | fmt.Println(ret) 90 | // Output: OK 91 | 92 | } 93 | 94 | // exemplifies the TdMin function 95 | func ExampleClient_TdMin() { 96 | host := "localhost:6379" 97 | var client = redisbloom.NewClient(host, "nohelp", nil) 98 | client.FlushAll() 99 | 100 | key := "example" 101 | _, err := client.TdCreate(key, 10) 102 | if err != nil { 103 | fmt.Println("Error:", err) 104 | } 105 | 106 | samples := map[float64]float64{1.0: 1.0, 2.0: 2.0, 3.0: 3.0} 107 | _, err = client.TdAdd(key, samples) 108 | if err != nil { 109 | fmt.Println("Error:", err) 110 | } 111 | 112 | min, err := client.TdMin(key) 113 | if err != nil { 114 | fmt.Println("Error:", err) 115 | } 116 | 117 | fmt.Println(min) 118 | // Output: 1 119 | } 120 | 121 | // exemplifies the TdMax function 122 | func ExampleClient_TdMax() { 123 | host := "localhost:6379" 124 | var client = redisbloom.NewClient(host, "nohelp", nil) 125 | client.FlushAll() 126 | 127 | key := "example" 128 | _, err := client.TdCreate(key, 10) 129 | if err != nil { 130 | fmt.Println("Error:", err) 131 | } 132 | 133 | samples := map[float64]float64{1.0: 1.0, 2.0: 2.0, 3.0: 3.0} 134 | _, err = client.TdAdd(key, samples) 135 | if err != nil { 136 | fmt.Println("Error:", err) 137 | } 138 | 139 | max, err := client.TdMax(key) 140 | if err != nil { 141 | fmt.Println("Error:", err) 142 | } 143 | 144 | fmt.Println(max) 145 | // Output: 3 146 | } 147 | 148 | // exemplifies the TdQuantile function 149 | func ExampleClient_TdQuantile() { 150 | host := "localhost:6379" 151 | var client = redisbloom.NewClient(host, "nohelp", nil) 152 | client.FlushAll() 153 | 154 | key := "example" 155 | _, err := client.TdCreate(key, 10) 156 | if err != nil { 157 | fmt.Println("Error:", err) 158 | } 159 | 160 | samples := map[float64]float64{1.0: 1.0, 2.0: 1.0, 3.0: 1.0, 4.0: 1.0, 5.0: 1.0} 161 | _, err = client.TdAdd(key, samples) 162 | if err != nil { 163 | fmt.Println("Error:", err) 164 | } 165 | 166 | ans, err := client.TdQuantile(key, 1.0) 167 | if err != nil { 168 | fmt.Println("Error:", err) 169 | } 170 | 171 | fmt.Println(ans) 172 | // Output: [5] 173 | } 174 | 175 | // exemplifies the TdCdf function 176 | func ExampleClient_TdCdf() { 177 | host := "localhost:6379" 178 | var client = redisbloom.NewClient(host, "nohelp", nil) 179 | client.FlushAll() 180 | 181 | key := "example" 182 | _, err := client.TdCreate(key, 10) 183 | if err != nil { 184 | fmt.Println("Error:", err) 185 | } 186 | 187 | samples := map[float64]float64{1.0: 1.0, 2.0: 1.0, 3.0: 1.0, 4.0: 1.0, 5.0: 1.0} 188 | _, err = client.TdAdd(key, samples) 189 | if err != nil { 190 | fmt.Println("Error:", err) 191 | } 192 | 193 | cdf, err := client.TdCdf(key, 1.0) 194 | if err != nil { 195 | fmt.Println("Error:", err) 196 | } 197 | 198 | fmt.Println(cdf) 199 | // Output: [0.2] 200 | } 201 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/RedisBloom/redisbloom-go 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/davecgh/go-spew v1.1.1 // indirect 7 | github.com/gomodule/redigo v1.8.9 8 | github.com/kr/text v0.2.0 // indirect 9 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect 10 | github.com/stretchr/testify v1.7.0 11 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect 12 | ) 13 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= 6 | github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= 7 | github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= 8 | github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= 9 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 10 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 11 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 12 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 13 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= 14 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= 15 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 16 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 17 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 18 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 19 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 20 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 21 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 22 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= 23 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 24 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 25 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 26 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 27 | -------------------------------------------------------------------------------- /pool.go: -------------------------------------------------------------------------------- 1 | package redis_bloom_go 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "sync" 7 | "time" 8 | 9 | "github.com/gomodule/redigo/redis" 10 | ) 11 | 12 | type ConnPool interface { 13 | Get() redis.Conn 14 | Close() error 15 | } 16 | 17 | type SingleHostPool struct { 18 | *redis.Pool 19 | } 20 | 21 | // 22 | //func (s SingleHostPool) Close() { 23 | // s.Pool.Close() 24 | //} 25 | 26 | func NewSingleHostPool(host string, authPass *string) *SingleHostPool { 27 | ret := &redis.Pool{ 28 | Dial: dialFuncWrapper(host, authPass), 29 | TestOnBorrow: testOnBorrow, 30 | MaxIdle: maxConns, 31 | } 32 | 33 | return &SingleHostPool{ret} 34 | } 35 | 36 | type MultiHostPool struct { 37 | sync.Mutex 38 | pools map[string]*redis.Pool 39 | hosts []string 40 | authPass *string 41 | } 42 | 43 | func (p *MultiHostPool) Close() (err error) { 44 | p.Lock() 45 | defer p.Unlock() 46 | for host, pool := range p.pools { 47 | poolErr := pool.Close() 48 | //preserve pool error if not nil but continue 49 | if poolErr != nil { 50 | if err == nil { 51 | err = fmt.Errorf("Error closing pool for host %s. Got %v.", host, poolErr) 52 | } else { 53 | err = fmt.Errorf("%v Error closing pool for host %s. Got %v.", err, host, poolErr) 54 | } 55 | } 56 | } 57 | return 58 | } 59 | 60 | func NewMultiHostPool(hosts []string, authPass *string) *MultiHostPool { 61 | return &MultiHostPool{ 62 | pools: make(map[string]*redis.Pool, len(hosts)), 63 | hosts: hosts, 64 | authPass: authPass, 65 | } 66 | } 67 | 68 | func (p *MultiHostPool) Get() redis.Conn { 69 | p.Lock() 70 | defer p.Unlock() 71 | 72 | host := p.hosts[rand.Intn(len(p.hosts))] 73 | pool, found := p.pools[host] 74 | 75 | if !found { 76 | pool = &redis.Pool{ 77 | Dial: dialFuncWrapper(host, p.authPass), 78 | TestOnBorrow: testOnBorrow, 79 | MaxIdle: maxConns, 80 | } 81 | p.pools[host] = pool 82 | } 83 | 84 | return pool.Get() 85 | } 86 | 87 | func dialFuncWrapper(host string, authPass *string) func() (redis.Conn, error) { 88 | return func() (redis.Conn, error) { 89 | conn, err := redis.Dial("tcp", host) 90 | if err != nil { 91 | return conn, err 92 | } 93 | if authPass != nil { 94 | _, err = conn.Do("AUTH", *authPass) 95 | } 96 | return conn, err 97 | } 98 | } 99 | 100 | func testOnBorrow(c redis.Conn, t time.Time) (err error) { 101 | if time.Since(t) > time.Minute { 102 | _, err = c.Do("PING") 103 | } 104 | return 105 | } 106 | -------------------------------------------------------------------------------- /pool_test.go: -------------------------------------------------------------------------------- 1 | package redis_bloom_go 2 | 3 | import ( 4 | "github.com/gomodule/redigo/redis" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestNewMultiHostPool(t *testing.T) { 10 | type args struct { 11 | hosts []string 12 | authPass *string 13 | } 14 | tests := []struct { 15 | name string 16 | args args 17 | wantPoolSize int 18 | wantConntNil bool 19 | }{ 20 | {"same connection string", args{[]string{"localhost:6379", "localhost:6379"}, nil}, 2, false}, 21 | } 22 | for _, tt := range tests { 23 | t.Run(tt.name, func(t *testing.T) { 24 | var got *MultiHostPool 25 | got = NewMultiHostPool(tt.args.hosts, tt.args.authPass) 26 | if len(got.hosts) != tt.wantPoolSize { 27 | t.Errorf("NewMultiHostPool() = %v, want %v", got, tt.wantPoolSize) 28 | } 29 | if gotConn := got.Get(); tt.wantConntNil == false && gotConn == nil { 30 | t.Errorf("NewMultiHostPool().Get() = %v, want %v", gotConn, tt.wantConntNil) 31 | } 32 | }) 33 | } 34 | } 35 | 36 | func TestMultiHostPool_Close(t *testing.T) { 37 | host, password := getTestConnectionDetails() 38 | // Test a simple flow 39 | if password == "" { 40 | oneMulti := NewMultiHostPool([]string{host}, nil) 41 | conn := oneMulti.Get() 42 | assert.NotNil(t, conn) 43 | err := oneMulti.Close() 44 | assert.Nil(t, err) 45 | err = oneMulti.Close() 46 | assert.NotNil(t, conn) 47 | severalMulti := NewMultiHostPool([]string{host, host}, nil) 48 | connMulti := severalMulti.Get() 49 | assert.NotNil(t, connMulti) 50 | err = severalMulti.Close() 51 | assert.Nil(t, err) 52 | } 53 | // Exhaustive test 54 | dial := func() (redis.Conn, error) { 55 | return redis.Dial("tcp", host, redis.DialPassword(password)) 56 | } 57 | pool1 := &redis.Pool{Dial: dial, MaxIdle: maxConns} 58 | pool2 := &redis.Pool{Dial: dial, MaxIdle: maxConns} 59 | pool3 := &redis.Pool{Dial: dial, MaxIdle: maxConns} 60 | //Close pull3 prior to enforce error 61 | pool3.Close() 62 | pool4 := &redis.Pool{Dial: dial, MaxIdle: maxConns} 63 | 64 | type fields struct { 65 | pools map[string]*redis.Pool 66 | hosts []string 67 | } 68 | tests := []struct { 69 | name string 70 | fields fields 71 | wantErr bool 72 | }{ 73 | {"empty", fields{map[string]*redis.Pool{}, []string{}}, false}, 74 | {"normal", fields{map[string]*redis.Pool{"hostpool1": pool1}, []string{"hostpool1"}}, false}, 75 | {"pool3-already-close", fields{map[string]*redis.Pool{"hostpool2": pool2, "hostpool3": pool3, "hostpool4": pool4}, []string{"hostpool2", "hostpool3", "hostpool3"}}, false}, 76 | } 77 | for _, tt := range tests { 78 | t.Run(tt.name, func(t *testing.T) { 79 | p := &MultiHostPool{ 80 | pools: tt.fields.pools, 81 | hosts: tt.fields.hosts, 82 | } 83 | if err := p.Close(); (err != nil) != tt.wantErr { 84 | t.Errorf("Close() error = %v, wantErr %v", err, tt.wantErr) 85 | } 86 | // ensure all connections are really closed 87 | if !tt.wantErr { 88 | for _, pool := range p.pools { 89 | if _, err := pool.Get().Do("PING"); err == nil { 90 | t.Errorf("expected error after connection closed. Got %v", err) 91 | } 92 | } 93 | } 94 | }) 95 | } 96 | } 97 | --------------------------------------------------------------------------------