├── .gitignore ├── LICENSE ├── README.md ├── Vagrantfile └── bloomd ├── client.go ├── client_test.go ├── connection.go ├── connection_test.go ├── error.go ├── filter.go ├── filter_test.go └── test_helpers.go /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .vagrant 3 | bloomd/go-bloomd.test 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Robby Colvin 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-bloomd [![Build Status](https://drone.io/github.com/geetarista/go-bloomd/status.png)](https://drone.io/github.com/geetarista/go-bloomd/latest) [![GoDoc](https://godoc.org/github.com/geetarista/go-bloomd/bloomd?status.svg)](https://godoc.org/github.com/geetarista/go-bloomd/bloomd) 2 | 3 | A [bloomd](https://github.com/armon/bloomd) client powered by [Go](http://golang.org). 4 | 5 | ## Installation 6 | 7 | ```bash 8 | go get github.com/geetarista/go-bloomd/bloomd 9 | ``` 10 | 11 | ## Testing 12 | 13 | I use Vagrant to run the tests against a BloomD server. Use the included [Vagrantfile](Vagrantfile) and make sure you use your VM's IP address in `test_helpers.go`. 14 | 15 | ## License 16 | 17 | MIT. See `LICENSE`. 18 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | $script = < bloomd.conf 17 | 18 | ps elf | grep -i bloomd | awk '{print "kill -9 "$2}' |sh 19 | 20 | bloomd -f /home/vagrant/bloomd.conf & 21 | echo "Done." 22 | EOF 23 | 24 | Vagrant.configure("2") do |config| 25 | config.vm.box = "hashicorp/precise64" 26 | config.vm.provision :shell, :inline => $script 27 | config.vm.network :public_network, :bridge => "en0: Wi-Fi (AirPort)" 28 | end 29 | -------------------------------------------------------------------------------- /bloomd/client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Provides a client abstraction around the BloomD interface. 3 | 4 | Example: 5 | client := bloomd.Client{Server: "10.0.0.30:8673"} 6 | filter := bloomd.Filter{Name: "coolfilter"} 7 | if err := bloomd.CreateFilter(filter); err != nil { 8 | // handle error 9 | } 10 | filters, _ := bloomd.ListFilters() 11 | fmt.Printf("%+v", filters["coolfilter"]) 12 | */ 13 | package bloomd 14 | 15 | import ( 16 | "strconv" 17 | "strings" 18 | ) 19 | 20 | // If using multiple BloomD servers, it is recommended to use a BloomD Ring 21 | // and only use the proxy as the Server field for your client. 22 | type Client struct { 23 | Server string 24 | Timeout int 25 | Conn *Connection 26 | ServerInfo string 27 | InfoTime int 28 | HashKeys bool 29 | } 30 | 31 | func NewClient(address string) Client { 32 | return Client{Server: address, Conn: &Connection{Server: address}} 33 | } 34 | 35 | func (c *Client) CreateFilter(f *Filter) error { 36 | if f.Prob > 0 && f.Capacity < 1 { 37 | return errInvalidCapacity 38 | } 39 | 40 | cmd := "create " + f.Name 41 | if f.Capacity > 0 { 42 | cmd = cmd + " capacity=" + strconv.Itoa(f.Capacity) 43 | } 44 | if f.Prob > 0 { 45 | cmd = cmd + " prob=" + strconv.FormatFloat(f.Prob, 'f', -1, 64) 46 | } 47 | if f.InMemory { 48 | cmd = cmd + " in_memory=1" 49 | } 50 | 51 | err := c.Conn.Send(cmd) 52 | if err != nil { 53 | return err 54 | } 55 | resp, err := c.Conn.Read() 56 | if err != nil { 57 | return err 58 | } 59 | if resp != "Done" && resp != "Exists" { 60 | return errInvalidResponse(resp) 61 | } 62 | f.Conn = c.Conn 63 | f.HashKeys = c.HashKeys 64 | return nil 65 | } 66 | 67 | func (c *Client) GetFilter(name string) *Filter { 68 | return &Filter{ 69 | Name: name, 70 | Conn: c.Conn, 71 | HashKeys: c.HashKeys, 72 | } 73 | } 74 | 75 | // Lists all the available filters 76 | func (c *Client) ListFilters() (responses map[string]string, err error) { 77 | err = c.Conn.Send("list") 78 | if err != nil { 79 | return 80 | } 81 | 82 | responses = make(map[string]string) 83 | resp, err := c.Conn.ReadBlock() 84 | if err != nil { 85 | return 86 | } 87 | for _, line := range resp { 88 | split := strings.SplitN(line, " ", 2) 89 | responses[split[0]] = split[1] 90 | } 91 | return responses, nil 92 | } 93 | 94 | // Instructs server to flush to disk 95 | func (c *Client) Flush() error { 96 | err := c.Conn.Send("flush") 97 | if err != nil { 98 | return err 99 | } 100 | resp, err := c.Conn.Read() 101 | if err != nil { 102 | return err 103 | } 104 | if resp != "DONE" { 105 | return err 106 | } 107 | return nil 108 | } 109 | -------------------------------------------------------------------------------- /bloomd/client_test.go: -------------------------------------------------------------------------------- 1 | package bloomd 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | // Clear everything out of bloomd before running tests. 9 | func TestDropEverything(t *testing.T) { 10 | client := NewClient(serverAddress) 11 | filters, _ := client.ListFilters() 12 | for f, _ := range filters { 13 | filter := Filter{Name: f, Conn: client.Conn} 14 | filter.Drop() 15 | } 16 | } 17 | 18 | func TestCreateFilter(t *testing.T) { 19 | client := NewClient(serverAddress) 20 | err := client.CreateFilter(&validFilter) 21 | failIfError(t, err) 22 | err = client.CreateFilter(&anotherFilter) 23 | failIfError(t, err) 24 | } 25 | 26 | func TestGetFilter(t *testing.T) { 27 | client := NewClient(serverAddress) 28 | filter := client.GetFilter(validFilter.Name) 29 | if filter.Name != validFilter.Name { 30 | t.Error("Name not equal") 31 | } 32 | if filter.HashKeys != validFilter.HashKeys { 33 | t.Error("HashKeys not equal") 34 | } 35 | } 36 | 37 | func TestListFilters(t *testing.T) { 38 | client := NewClient(serverAddress) 39 | filters, err := client.ListFilters() 40 | failIfError(t, err) 41 | if filters[validFilter.Name] == "" { 42 | fmt.Printf("%+v\n", filters) 43 | t.Error(validFilter.Name) 44 | } 45 | } 46 | 47 | func TestClientFlush(t *testing.T) { 48 | client := NewClient(serverAddress) 49 | err := client.Flush() 50 | failIfError(t, err) 51 | } 52 | -------------------------------------------------------------------------------- /bloomd/connection.go: -------------------------------------------------------------------------------- 1 | package bloomd 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "net" 7 | "os" 8 | "strconv" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | type Connection struct { 14 | Server string 15 | Timeout time.Duration 16 | Socket *net.TCPConn 17 | File *os.File 18 | Attempts int 19 | Reader *bufio.Reader 20 | } 21 | 22 | // Create a TCP socket for the connection 23 | func (c *Connection) createSocket() (err error) { 24 | addr, err := net.ResolveTCPAddr("tcp", c.Server) 25 | if err != nil { 26 | return err 27 | } 28 | c.Socket, err = net.DialTCP("tcp", nil, addr) 29 | if err != nil { 30 | return err 31 | } 32 | c.Reader = bufio.NewReader(c.Socket) 33 | if c.Attempts == 0 { 34 | c.Attempts = 3 35 | } 36 | return nil 37 | } 38 | 39 | // Sends a command to the server 40 | func (c *Connection) Send(cmd string) error { 41 | if c.Socket == nil || c.Socket.LocalAddr() == nil { 42 | err := c.createSocket() 43 | if err != nil { 44 | return &BloomdError{ErrorString: err.Error()} 45 | } 46 | } 47 | for i := 0; i < c.Attempts; i++ { 48 | _, err := c.Socket.Write([]byte(cmd + "\n")) 49 | 50 | if err != nil { 51 | c.createSocket() 52 | break 53 | } 54 | return nil 55 | } 56 | 57 | return errSendFailed(cmd, strconv.Itoa(c.Attempts)) 58 | } 59 | 60 | // Returns a single line from the socket file 61 | func (c *Connection) Read() (line string, err error) { 62 | if c.Socket == nil || c.Socket.LocalAddr() == nil { 63 | err := c.createSocket() 64 | if err != nil { 65 | return "", &BloomdError{ErrorString: err.Error()} 66 | } 67 | } 68 | 69 | l, rerr := c.Reader.ReadString('\n') 70 | if rerr != nil && rerr != io.EOF { 71 | return l, &BloomdError{ErrorString: rerr.Error()} 72 | } 73 | return strings.TrimRight(l, "\r\n"), nil 74 | } 75 | 76 | // Reads a response block from the server. The servers responses are between 77 | // `start` and `end` which can be optionally provided. Returns an array of 78 | // the lines within the block. 79 | func (c *Connection) ReadBlock() (lines []string, err error) { 80 | first, err := c.Read() 81 | if err != nil { 82 | return lines, err 83 | } 84 | if first != "START" { 85 | return lines, &BloomdError{ErrorString: "Did not get block start START! Got '" + string(first) + "'!"} 86 | } 87 | 88 | for { 89 | line, err := c.Read() 90 | if err != nil { 91 | return lines, err 92 | } 93 | if line == "END" || line == "" { 94 | break 95 | } 96 | lines = append(lines, string(line)) 97 | } 98 | return lines, nil 99 | } 100 | 101 | // Convenience wrapper around `send` and `read`. Sends a command, 102 | // and reads the response, performing a retry if necessary. 103 | func (c *Connection) SendAndReceive(cmd string) (string, error) { 104 | err := c.Send(cmd) 105 | if err != nil { 106 | return "", err 107 | } 108 | return c.Read() 109 | } 110 | 111 | func (c *Connection) responseBlockToMap() (map[string]string, error) { 112 | lines, err := c.ReadBlock() 113 | if err != nil { 114 | return nil, err 115 | } 116 | theMap := make(map[string]string) 117 | for _, line := range lines { 118 | split := strings.SplitN(line, " ", 2) 119 | theMap[split[0]] = split[1] 120 | } 121 | return theMap, nil 122 | } 123 | -------------------------------------------------------------------------------- /bloomd/connection_test.go: -------------------------------------------------------------------------------- 1 | package bloomd 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestCreateSocket(t *testing.T) { 8 | conn := Connection{Server: serverAddress} 9 | conn.createSocket() 10 | if conn.Socket == nil { 11 | t.Fail() 12 | } 13 | } 14 | 15 | func TestSend(t *testing.T) { 16 | conn := Connection{Server: serverAddress} 17 | err := conn.Send("derp") 18 | failIfError(t, err) 19 | } 20 | 21 | func TestReadDerp(t *testing.T) { 22 | conn := Connection{Server: serverAddress} 23 | err := conn.Send("list") 24 | failIfError(t, err) 25 | resp, err := conn.Read() 26 | failIfError(t, err) 27 | if resp != "START" { 28 | t.Error("Got: " + resp) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /bloomd/error.go: -------------------------------------------------------------------------------- 1 | package bloomd 2 | 3 | type BloomdError struct { 4 | ErrorString string 5 | } 6 | 7 | func (err *BloomdError) Error() string { 8 | return err.ErrorString 9 | } 10 | 11 | var ( 12 | errInvalidCapacity = &BloomdError{"Must provide size with probability!"} 13 | ) 14 | 15 | func errInvalidResponse(resp string) error { 16 | return &BloomdError{ErrorString: "Got response: " + resp} 17 | } 18 | 19 | func errCommandFailed(cmd, attempt string) error { 20 | return &BloomdError{ 21 | ErrorString: "Failed to send command to bloomd server: " + cmd + ". Attempt: " + attempt, 22 | } 23 | } 24 | 25 | func errSendFailed(cmd, attempts string) error { 26 | return &BloomdError{ 27 | ErrorString: "Failed to send command '" + cmd + "' after " + attempts + " attempts!", 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /bloomd/filter.go: -------------------------------------------------------------------------------- 1 | // Provides an interface to a single Bloomd filter 2 | 3 | package bloomd 4 | 5 | import ( 6 | "crypto/sha1" 7 | "fmt" 8 | "strings" 9 | ) 10 | 11 | type Filter struct { 12 | Name string 13 | Conn *Connection 14 | HashKeys bool 15 | // Optional 16 | Capacity int // The initial capacity of the filter 17 | Prob float64 // The inital probability of false positives 18 | InMemory bool // If True, specified that the filter should be created 19 | } 20 | 21 | // Returns the key we should send to the server 22 | func (f *Filter) getKey(key string) string { 23 | if f.HashKeys { 24 | h := sha1.New() 25 | s := h.Sum([]byte(key)) 26 | return fmt.Sprintf("%x", s) 27 | } 28 | return key 29 | } 30 | 31 | // Adds a new key to the filter. Returns True/False if the key was added 32 | func (f *Filter) Set(key string) (bool, error) { 33 | cmd := "s " + f.Name + " " + f.getKey(key) 34 | resp, err := f.Conn.SendAndReceive(cmd) 35 | if err != nil { 36 | return false, err 37 | } 38 | if resp == "Yes" || resp == "No" { 39 | return resp == "Yes", nil 40 | } 41 | return false, errInvalidResponse(resp) 42 | } 43 | 44 | func (f *Filter) groupCommand(kind string, keys []string) (rs []bool, e error) { 45 | cmd := kind + " " + f.Name 46 | for _, key := range keys { 47 | cmd = cmd + " " + f.getKey(key) 48 | } 49 | resp, e := f.Conn.SendAndReceive(cmd) 50 | if e != nil { 51 | return rs, &BloomdError{ErrorString: e.Error()} 52 | } 53 | if strings.HasPrefix(resp, "Yes") || strings.HasPrefix(resp, "No") { 54 | split := strings.Split(resp, " ") 55 | for _, res := range split { 56 | rs = append(rs, res == "Yes") 57 | } 58 | } 59 | return rs, nil 60 | } 61 | 62 | // Performs a bulk set command, adds multiple keys in the filter 63 | func (f *Filter) Bulk(keys []string) (responses []bool, err error) { 64 | return f.groupCommand("b", keys) 65 | } 66 | 67 | // Performs a multi command, checks for multiple keys in the filter 68 | func (f *Filter) Multi(keys []string) (responses []bool, err error) { 69 | return f.groupCommand("m", keys) 70 | } 71 | 72 | func (f *Filter) sendCommand(cmd string) error { 73 | resp, err := f.Conn.SendAndReceive(cmd + " " + f.Name) 74 | if err != nil { 75 | return err 76 | } 77 | if resp != "Done" { 78 | return errInvalidResponse(resp) 79 | } 80 | return nil 81 | } 82 | 83 | // Deletes the filter permanently from the server 84 | func (f *Filter) Drop() error { 85 | return f.sendCommand("drop") 86 | } 87 | 88 | // Closes the filter on the server 89 | func (f *Filter) Close() error { 90 | return f.sendCommand("close") 91 | } 92 | 93 | // Clears the filter on the server 94 | func (f *Filter) Clear() error { 95 | return f.sendCommand("clear") 96 | } 97 | 98 | // Forces the filter to flush to disk 99 | func (f *Filter) Flush() error { 100 | return f.sendCommand("flush") 101 | } 102 | 103 | // Returns the info dictionary about the filter 104 | func (f *Filter) Info() (map[string]string, error) { 105 | if err := f.Conn.Send("info " + f.Name); err != nil { 106 | return nil, err 107 | } 108 | info, err := f.Conn.responseBlockToMap() 109 | if err != nil { 110 | return nil, err 111 | } 112 | return info, nil 113 | } 114 | -------------------------------------------------------------------------------- /bloomd/filter_test.go: -------------------------------------------------------------------------------- 1 | package bloomd 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | var TestHash = "61736466da39a3ee5e6b4b0d3255bfef95601890afd80709" 8 | 9 | func TestGetKeyHash(t *testing.T) { 10 | dummyFilter.HashKeys = true 11 | k := dummyFilter.getKey("asdf") 12 | if k != TestHash { 13 | t.Fail() 14 | } 15 | } 16 | 17 | func TestGetKeyString(t *testing.T) { 18 | dummyFilter.HashKeys = false 19 | k := dummyFilter.getKey("asdf") 20 | if k != "asdf" { 21 | t.Fail() 22 | } 23 | } 24 | 25 | func TestSet(t *testing.T) { 26 | ok, err := validFilter.Set("derp") 27 | failIfError(t, err) 28 | if ok != true { 29 | t.Error(ok) 30 | } 31 | } 32 | 33 | func TestBulk(t *testing.T) { 34 | responses, err := validFilter.Bulk([]string{"herp", "derpina"}) 35 | failIfError(t, err) 36 | for _, response := range responses { 37 | if response != true { 38 | t.Error("Bulk fail") 39 | } 40 | } 41 | } 42 | 43 | func TestMulti(t *testing.T) { 44 | responses, err := validFilter.Multi([]string{"derp", "herp", "derpina"}) 45 | failIfError(t, err) 46 | for _, response := range responses { 47 | if response != true { 48 | t.Error("Multi fail") 49 | } 50 | } 51 | } 52 | 53 | func TestClose(t *testing.T) { 54 | err := validFilter.Close() 55 | failIfError(t, err) 56 | err = anotherFilter.Close() 57 | failIfError(t, err) 58 | } 59 | 60 | func TestClear(t *testing.T) { 61 | err := anotherFilter.Clear() 62 | failIfError(t, err) 63 | } 64 | 65 | func TestFilterFlush(t *testing.T) { 66 | err := validFilter.Flush() 67 | failIfError(t, err) 68 | } 69 | 70 | func TestInfo(t *testing.T) { 71 | info, err := validFilter.Info() 72 | failIfError(t, err) 73 | for _, field := range infoFields { 74 | if info[field] == "" { 75 | t.Error(field + " not in info") 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /bloomd/test_helpers.go: -------------------------------------------------------------------------------- 1 | package bloomd 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | var ( 8 | serverHost = "127.0.0.1" 9 | serverPort = "8673" 10 | serverAddress = serverHost + ":" + serverPort 11 | dummyFilter = Filter{Name: "asdf"} 12 | validFilter = Filter{ 13 | Name: "thing", 14 | InMemory: true, 15 | Conn: &Connection{Server: serverAddress}, 16 | } 17 | anotherFilter = Filter{ 18 | Name: "another", 19 | Conn: &Connection{Server: serverAddress}, 20 | } 21 | infoFields = []string{"storage", "check_hits", "in_memory", "page_outs", 22 | "page_ins", "size", "check_misses", "capacity", "sets", "checks", 23 | "set_misses", "set_hits", "probability"} 24 | ) 25 | 26 | func failIfError(t *testing.T, err error) { 27 | if err != nil { 28 | t.Error(err.Error()) 29 | } 30 | } 31 | --------------------------------------------------------------------------------