├── README.md ├── config.go ├── debian ├── changelog ├── compat ├── control ├── copyright ├── mcsauna.conf ├── mcsauna.install ├── mcsauna.postinst ├── mcsauna.upstart ├── rules └── source │ └── format ├── hot_keys.go ├── hot_keys_test.go ├── main.go ├── memcached.go ├── memcached_test.go ├── regexp.go └── regexp_test.go /README.md: -------------------------------------------------------------------------------- 1 | # mcsauna 2 | 3 | mcsauna allows you to track the hottest keys on your memcached instances, 4 | reporting back in a graphite-friendly format. Regexps can be specified to 5 | group similar keys into the same bucket, for high-cardinality memcached 6 | instances, or tracking lots of keys over time. 7 | 8 | Key rates are reported in the format: 9 | 10 | mcsauna.keys.foo: 3 11 | 12 | Errors in processing are reported in the format: 13 | 14 | mcsauna.errors.bar: 3 15 | 16 | If you are using diamond, you can output these to a file and watch via 17 | [FilesCollector](http://diamond.readthedocs.io/en/latest/collectors/FilesCollector/). 18 | 19 | Note that at the moment, TCP reassembly / reordering is not supported. This 20 | should only be a problem for the case of multigets that span more than one 21 | packet. In these cases, an error will be reported indicating the command was 22 | truncated. 23 | 24 | ## Arguments 25 | 26 | $ ./mcsauna --help 27 | Usage of ./mcsauna: 28 | -c string 29 | config file 30 | -e show errors in parsing as a metric (default true) 31 | -i string 32 | capture interface (default "any") 33 | -n int 34 | reporting interval (seconds, default 5) 35 | -p int 36 | capture port (default 11211) 37 | -q suppress stdout output (default false) 38 | -r int 39 | number of items to report (default 20) 40 | -w string 41 | file to write output to 42 | 43 | 44 | ## Configuration 45 | 46 | All command-line options can be specified via a configuration file in json 47 | format. Regular expressions and related options can only be specified in 48 | config. Command-line arguments will override settings passed in 49 | configuration. 50 | 51 | Pass a configuration file using `-c`: 52 | 53 | # ./mcsauna -c conf.json 54 | 55 | Example configuration: 56 | 57 | { 58 | "regexps": [ 59 | {"re": "^Foo_[0-9]+$", "name": "foo"}, 60 | {"re": "^Bar_[0-9]+$", "name": "bar"}, 61 | {"re": "^Baz_[0-9]+$", "name" "baz"}, 62 | ], 63 | "interval": 5, 64 | "interface": "eth0", 65 | "port": 11211, 66 | "quiet": false, 67 | "show_errors": true, 68 | "output_file": "/tmp/mcsauna.out" 69 | } 70 | 71 | If regexps are specified, individual hot keys will not be reported. If not 72 | specifying regular expressions, you can limit the number of items that will 73 | be reported: 74 | 75 | { 76 | "interval": 5, 77 | "num_items_to_report": 20 78 | } 79 | 80 | When debugging regular expressions, you can see which keys did not match 81 | with the `show_unmatched` flag set to `true`. 82 | 83 | ## Known Issues 84 | 85 | The attempt to add support for multiple commands per packet caused a 86 | performance regression that hasn't been addressed. If you're experiencing 87 | this, version 1.0.2 (3dbb6d5179448f40c183cbb489c07d0862b8e57a) is recommended. 88 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | ) 7 | 8 | type RegexpConfig struct { 9 | Name string `json:"name"` 10 | Re string `json:"re"` 11 | } 12 | 13 | type Config struct { 14 | Regexps []RegexpConfig `json:"regexps"` 15 | Interval int `json:"interval"` 16 | Interface string `json:"interface"` 17 | Port int `json:"port"` 18 | NumItemsToReport int `json:"num_items_to_report"` 19 | Quiet bool `json:"quiet"` 20 | OutputFile string `json:"output_file"` 21 | ShowErrors bool `json:"show_errors"` 22 | 23 | /* When using regexps, include a list of keys that did not match in the 24 | * output. Useful for debugging regular expressions. 25 | */ 26 | ShowUnmatched bool `json:"show_unmatched"` 27 | } 28 | 29 | func NewConfig(config_data []byte) (config Config, err error) { 30 | config = Config{ 31 | Regexps: []RegexpConfig{}, 32 | Interval: 5, 33 | Interface: "any", 34 | Port: 11211, 35 | NumItemsToReport: 20, 36 | Quiet: false, 37 | ShowErrors: true, 38 | ShowUnmatched: false, 39 | } 40 | err = json.Unmarshal(config_data, &config) 41 | if err != nil { 42 | return config, err 43 | } 44 | 45 | // Validate config 46 | for _, regexp_config := range config.Regexps { 47 | if regexp_config.Name == "" || regexp_config.Re == "" { 48 | return config, errors.New( 49 | "Config error: regular expressions must have both a 're' and 'name' field.") 50 | } 51 | } 52 | 53 | return config, nil 54 | } 55 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | mcsauna (1.0.4) precise trusty; urgency=low 2 | 3 | * fix issue with truncated packets introduced by 1.0.3 4 | 5 | -- Daniel Ellis Wed, 07 Sep 2016 18:13:33 -0700 6 | 7 | mcsauna (1.0.3) precise trusty; urgency=low 8 | 9 | * add support for multiple commands per packet 10 | 11 | -- Daniel Ellis Tue, 06 Sep 2016 12:53:33 -0700 12 | 13 | mcsauna (1.0.2) precise trusty; urgency=low 14 | 15 | * add support for incr and decr 16 | * output all errors as a stat, not just truncations 17 | 18 | -- Daniel Ellis Thu, 01 Sep 2016 11:53:33 -0700 19 | 20 | mcsauna (1.0.1) precise trusty; urgency=low 21 | 22 | * add ability to include keys unmatched by regular expressions in output 23 | * do not limit number of output lines when regular expressions are specified 24 | 25 | -- Daniel Ellis Mon, 29 Aug 2016 14:53:33 -0700 26 | 27 | mcsauna (1.0.0) precise trusty; urgency=low 28 | 29 | * initial release 30 | 31 | -- Daniel Ellis Mon, 15 Aug 2016 10:53:33 -0700 32 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 9 2 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: mcsauna 2 | Maintainer: Daniel Ellis 3 | Section: devel 4 | Priority: optional 5 | Build-Depends: debhelper (>= 7.0) 6 | Standards-Version: 3.9.3 7 | Homepage: http://github.com/reddit/mcsauna 8 | Vcs-Git: git://github.com/reddit/mcsauna.git 9 | 10 | Package: mcsauna 11 | Architecture: amd64 12 | Depends: ${misc:Depends}, libcap2-bin 13 | Description: Memcached hot key tracker and stats generator 14 | Watches the network via pcap and outputs graphite-friendly stats of hot keys. 15 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: mcsauna 3 | Source: https://github.com/reddit/mcsauna 4 | 5 | Files: * 6 | Copyright: 2016 reddit Inc. 7 | License: Expat 8 | 9 | Files: debian/* 10 | Copyright: 2016 reddit Inc. 11 | License: Expat 12 | 13 | License: Expat 14 | Permission is hereby granted, free of charge, to any person obtaining 15 | a copy of this software and associated documentation files (the 16 | "Software"), to deal in the Software without restriction, including 17 | without limitation the rights to use, copy, modify, merge, publish, 18 | distribute, sublicense, and/or sell copies of the Software, and to 19 | permit persons to whom the Software is furnished to do so, subject to 20 | the following conditions: 21 | . 22 | The above copyright notice and this permission notice shall be included 23 | in all copies or substantial portions of the Software. 24 | . 25 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 28 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 29 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 30 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 31 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 32 | -------------------------------------------------------------------------------- /debian/mcsauna.conf: -------------------------------------------------------------------------------- 1 | { 2 | "regexps": [], 3 | "interval": 5, 4 | "quiet": true, 5 | "show_errors": true, 6 | "output_file": "/var/lib/mcsauna/mcsauna.out" 7 | } 8 | -------------------------------------------------------------------------------- /debian/mcsauna.install: -------------------------------------------------------------------------------- 1 | debian/mcsauna.conf /etc/mcsauna 2 | -------------------------------------------------------------------------------- /debian/mcsauna.postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | case "$1" in 6 | configure) 7 | if ! getent passwd Debian-mcsauna > /dev/null ; then 8 | 9 | echo 'Adding system-user for mcsauna' 1>&2 10 | adduser --system --group --quiet --home /var/spool/mcsauna \ 11 | --no-create-home --disabled-login --force-badname Debian-mcsauna 12 | fi 13 | 14 | install -d -oDebian-mcsauna -gDebian-mcsauna -m755 /var/lib/mcsauna/ 15 | setcap cap_net_raw,cap_setpcap=ep /usr/bin/mcsauna 16 | ;; 17 | esac 18 | 19 | #DEBHELPER# 20 | -------------------------------------------------------------------------------- /debian/mcsauna.upstart: -------------------------------------------------------------------------------- 1 | description "daemon to report back hot memcached keys in a graphite-friendly format" 2 | 3 | start on networking 4 | stop on runlevel [!2345] 5 | 6 | respawn 7 | respawn limit 10 5 8 | 9 | setuid Debian-mcsauna 10 | 11 | exec /usr/bin/mcsauna -c /etc/mcsauna/mcsauna.conf 12 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | GOPKG = github.com/reddit/mcsauna 4 | 5 | # These paths are currently hacks to get around the ancient golang 6 | # ubuntu package and the lack of and available package for the google pcap 7 | # library, which this depends on. They should be changed to fit your local 8 | # build environment. 9 | # 10 | # Built with go 1.6.3: 11 | # 12 | # https://storage.googleapis.com/golang/go1.6.3.linux-amd64.tar.gz 13 | GOPATH = "/home/vagrant/work" 14 | GOBIN = "/usr/local/go/bin/go" 15 | 16 | %: 17 | dh $@ 18 | 19 | clean: 20 | dh_clean 21 | rm -f mcsauna.debhelper.log 22 | 23 | binary-arch: clean 24 | dh_prep 25 | dh_installdirs /usr/bin 26 | mkdir -p "${GOPATH}/src/${GOPKG}" 27 | find . -path ./debian -prune -o -type f -name '*.go' -exec tar cf - {} + \ 28 | | (cd "${GOPATH}/src/${GOPKG}" && tar xvf -) 29 | GOPATH=${GOPATH} ${GOBIN} build -v -o $(CURDIR)/debian/tmp/bin/mcsauna ${GOPKG} 30 | GOPATH=${GOPATH} ${GOBIN} test ${GOPKG} 31 | dh_install bin/mcsauna /usr/bin 32 | dh_strip 33 | dh_installinit 34 | dh_installchangelogs 35 | dh_installdocs 36 | dh_installdeb 37 | dh_compress 38 | dh_fixperms 39 | dh_gencontrol 40 | dh_md5sums 41 | dh_builddeb 42 | 43 | binary: binary-arch 44 | 45 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (native) 2 | -------------------------------------------------------------------------------- /hot_keys.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "container/heap" 5 | "sync" 6 | ) 7 | 8 | type Key struct { 9 | Name string 10 | Hits int 11 | } 12 | 13 | // KeyHeap keeps track of hot keys and pops them off ordered by hotness, 14 | // greatest hotness first. 15 | type KeyHeap []*Key 16 | 17 | func (h KeyHeap) Len() int { return len(h) } 18 | 19 | // Less sorts in reverse order so we will pop the hottest keys first 20 | // (i.e. we use > rather than <). 21 | func (h KeyHeap) Less(i, j int) bool { return h[i].Hits > h[j].Hits } 22 | 23 | func (h KeyHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } 24 | 25 | func (h *KeyHeap) Push(x interface{}) { 26 | *h = append(*h, x.(*Key)) 27 | } 28 | 29 | func (h *KeyHeap) Pop() interface{} { 30 | old := *h 31 | n := len(old) 32 | x := old[n-1] 33 | *h = old[0 : n-1] 34 | return x 35 | } 36 | 37 | type HotKeyPool struct { 38 | Lock sync.Mutex 39 | 40 | // Map of keys to hits 41 | items map[string]int 42 | } 43 | 44 | func NewHotKeyPool() *HotKeyPool { 45 | h := &HotKeyPool{} 46 | h.items = make(map[string]int) 47 | return h 48 | } 49 | 50 | // Add adds a new key to the hit counter or increments the key's hit counter 51 | // if it is already present. 52 | func (h *HotKeyPool) Add(keys []string) { 53 | h.Lock.Lock() 54 | defer h.Lock.Unlock() 55 | 56 | for _, key := range keys { 57 | if _, ok := h.items[key]; ok { 58 | h.items[key] += 1 59 | } else { 60 | h.items[key] = 1 61 | } 62 | 63 | } 64 | } 65 | 66 | // GetTopKeys returns a KeyHeap object. Keys can be popped from the 67 | // resulting object and will be ordered by hits, descending. 68 | func (h *HotKeyPool) GetTopKeys() *KeyHeap { 69 | h.Lock.Lock() 70 | defer h.Lock.Unlock() 71 | 72 | top_keys := &KeyHeap{} 73 | heap.Init(top_keys) 74 | 75 | for key, hits := range h.items { 76 | heap.Push(top_keys, &Key{key, hits}) 77 | } 78 | return top_keys 79 | } 80 | 81 | func (h *HotKeyPool) GetHits(key string) int { 82 | h.Lock.Lock() 83 | defer h.Lock.Unlock() 84 | return h.items[key] 85 | } 86 | 87 | // Rotate clears the data on the existing HotKeyPool, returning a new pool 88 | // containing the old data. This allows sorting and reporting to happen in 89 | // another goroutine, while counting can continue on new keys. 90 | func (h *HotKeyPool) Rotate() *HotKeyPool { 91 | h.Lock.Lock() 92 | defer h.Lock.Unlock() 93 | 94 | // Clone existing 95 | new_hot_key_pool := NewHotKeyPool() 96 | new_hot_key_pool.items = h.items 97 | 98 | // Clear existing values 99 | h.items = make(map[string]int) 100 | return new_hot_key_pool 101 | } 102 | -------------------------------------------------------------------------------- /hot_keys_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "container/heap" 5 | "testing" 6 | ) 7 | 8 | func stringsEqual(first []string, second []string) bool { 9 | if len(first) != len(second) { 10 | return false 11 | } 12 | for i := range first { 13 | if first[i] != second[i] { 14 | return false 15 | } 16 | } 17 | return true 18 | } 19 | 20 | func TestHotKeys(t *testing.T) { 21 | h := NewHotKeyPool() 22 | h.Add([]string{"foo", "foo", "baz", "baz", "bar", "baz"}) 23 | top_keys := h.GetTopKeys() 24 | expected_keys := []Key{ 25 | Key{"baz", 3}, 26 | Key{"foo", 2}, 27 | Key{"bar", 1}, 28 | } 29 | for _, key := range expected_keys { 30 | popped_key := heap.Pop(top_keys).(*Key) 31 | if key.Name != popped_key.Name { 32 | t.Errorf("Expected top key %v, got %v\n", key.Name, popped_key.Name) 33 | } 34 | if key.Hits != popped_key.Hits { 35 | t.Errorf("Expected key %s to have %d hits, got %vdn", 36 | key.Name, key.Hits, popped_key.Hits) 37 | } 38 | } 39 | } 40 | 41 | func TestHotKeysClone(t *testing.T) { 42 | h := NewHotKeyPool() 43 | h.Add([]string{"foo", "foo", "bar", "baz", "baz", "baz"}) 44 | 45 | rotated := h.Rotate() 46 | 47 | // Validate old top keys 48 | top_keys := rotated.GetTopKeys() 49 | expected_keys := []string{"baz", "foo", "bar"} 50 | for _, key := range expected_keys { 51 | popped_key := heap.Pop(top_keys).(*Key) 52 | if key != popped_key.Name { 53 | t.Errorf("Expected top key %v, got %v\n", key, popped_key) 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "container/heap" 5 | "flag" 6 | "fmt" 7 | "github.com/google/gopacket" 8 | "github.com/google/gopacket/pcap" 9 | "io/ioutil" 10 | "time" 11 | ) 12 | 13 | const CAPTURE_SIZE = 9000 14 | 15 | // startReportingLoop starts a loop that will periodically output statistics 16 | // on the hottest keys, and optionally, errors that occured in parsing. 17 | func startReportingLoop(config Config, hot_keys *HotKeyPool, errors *HotKeyPool) { 18 | sleep_duration := time.Duration(config.Interval) * time.Second 19 | time.Sleep(sleep_duration) 20 | for { 21 | st := time.Now() 22 | rotated_keys := hot_keys.Rotate() 23 | top_keys := rotated_keys.GetTopKeys() 24 | rotated_errors := errors.Rotate() 25 | top_errors := rotated_errors.GetTopKeys() 26 | 27 | // Build output 28 | output := "" 29 | /* Show keys */ 30 | i := 0 31 | for { 32 | if top_keys.Len() == 0 { 33 | break 34 | } 35 | 36 | /* Check if we've reached the specified key limit, but only if 37 | * the user didn't specify regular expressions to match on. */ 38 | if len(config.Regexps) == 0 && i >= config.NumItemsToReport { 39 | break 40 | } 41 | 42 | key := heap.Pop(top_keys) 43 | output += fmt.Sprintf("mcsauna.keys.%s %d\n", key.(*Key).Name, key.(*Key).Hits) 44 | 45 | i += 1 46 | } 47 | /* Show errors */ 48 | if config.ShowErrors { 49 | for top_errors.Len() > 0 { 50 | err := heap.Pop(top_errors) 51 | output += fmt.Sprintf( 52 | "mcsauna.errors.%s %d\n", err.(*Key).Name, err.(*Key).Hits) 53 | } 54 | } 55 | 56 | // Write to stdout 57 | if !config.Quiet { 58 | fmt.Print(output) 59 | } 60 | 61 | // Write to file 62 | if config.OutputFile != "" { 63 | err := ioutil.WriteFile(config.OutputFile, []byte(output), 0666) 64 | if err != nil { 65 | panic(err) 66 | } 67 | } 68 | 69 | elapsed := time.Now().Sub(st) 70 | time.Sleep(sleep_duration - elapsed) 71 | } 72 | } 73 | 74 | func main() { 75 | config_file := flag.String("c", "", "config file") 76 | interval := flag.Int("n", 0, "reporting interval (seconds, default 5)") 77 | network_interface := flag.String("i", "", "capture interface (default any)") 78 | port := flag.Int("p", 0, "capture port (default 11211)") 79 | num_items_to_report := flag.Int("r", 0, "number of items to report (default 20)") 80 | quiet := flag.Bool("q", false, "suppress stdout output (default false)") 81 | output_file := flag.String("w", "", "file to write output to") 82 | show_errors := flag.Bool("e", true, "show errors in parsing as a metric") 83 | flag.Parse() 84 | 85 | // Parse Config 86 | var config Config 87 | var err error 88 | if *config_file != "" { 89 | config_data, _ := ioutil.ReadFile(*config_file) 90 | config, err = NewConfig(config_data) 91 | if err != nil { 92 | panic(err) 93 | } 94 | } else { 95 | config, err = NewConfig([]byte{}) 96 | } 97 | 98 | // Parse CLI Args 99 | if *interval != 0 { 100 | config.Interval = *interval 101 | } 102 | if *network_interface != "" { 103 | config.Interface = *network_interface 104 | } 105 | if *port != 0 { 106 | config.Port = *port 107 | } 108 | if *num_items_to_report != 0 { 109 | config.NumItemsToReport = *num_items_to_report 110 | } 111 | if *quiet != false { 112 | config.Quiet = *quiet 113 | } 114 | if *output_file != "" { 115 | config.OutputFile = *output_file 116 | } 117 | if *show_errors != true { 118 | config.ShowErrors = *show_errors 119 | } 120 | 121 | // Build Regexps 122 | regexp_keys := NewRegexpKeys() 123 | for _, re := range config.Regexps { 124 | regexp_key, err := NewRegexpKey(re.Re, re.Name) 125 | if err != nil { 126 | panic(err) 127 | } 128 | regexp_keys.Add(regexp_key) 129 | } 130 | 131 | hot_keys := NewHotKeyPool() 132 | errors := NewHotKeyPool() 133 | 134 | // Setup pcap 135 | handle, err := pcap.OpenLive(config.Interface, CAPTURE_SIZE, true, pcap.BlockForever) 136 | if err != nil { 137 | panic(err) 138 | } 139 | filter := fmt.Sprintf("tcp and dst port %d", config.Port) 140 | err = handle.SetBPFFilter(filter) 141 | if err != nil { 142 | panic(err) 143 | } 144 | packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) 145 | 146 | go startReportingLoop(config, hot_keys, errors) 147 | 148 | // Grab a packet 149 | var ( 150 | payload []byte 151 | keys []string 152 | cmd_err int 153 | ) 154 | for packet := range packetSource.Packets() { 155 | app_data := packet.ApplicationLayer() 156 | if app_data == nil { 157 | continue 158 | } 159 | payload = app_data.Payload() 160 | 161 | // Process data 162 | prev_payload_len := 0 163 | for len(payload) > 0 { 164 | _, keys, payload, cmd_err = parseCommand(payload) 165 | 166 | // ... We keep track of the payload length to make sure we don't end 167 | // ... up in an infinite loop if one of the processors repeatedly 168 | // ... sends us the same remainder. This should never happen, but 169 | // ... if it does, it would be better to move on to the next packet 170 | // ... rather than spin CPU doing nothing. 171 | if len(payload) == prev_payload_len { 172 | break 173 | } 174 | prev_payload_len = len(payload) 175 | 176 | if cmd_err == ERR_NONE { 177 | 178 | // Raw key 179 | if len(config.Regexps) == 0 { 180 | hot_keys.Add(keys) 181 | } else { 182 | 183 | // Regex 184 | matches := []string{} 185 | match_errors := []string{} 186 | for _, key := range keys { 187 | matched_regex, err := regexp_keys.Match(key) 188 | if err != nil { 189 | match_errors = append(match_errors, "match_error") 190 | 191 | // The user has requested that we also show keys that 192 | // weren't matched at all, probably for debugging. 193 | if config.ShowUnmatched { 194 | matches = append(matches, key) 195 | } 196 | 197 | } else { 198 | matches = append(matches, matched_regex) 199 | } 200 | } 201 | hot_keys.Add(matches) 202 | errors.Add(match_errors) 203 | } 204 | } else { 205 | errors.Add([]string{ERR_TO_STAT[cmd_err]}) 206 | } 207 | } 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /memcached.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | const ( 10 | ERR_NONE = iota 11 | ERR_NO_CMD 12 | ERR_INVALID_CMD 13 | ERR_TRUNCATED 14 | ERR_INCOMPLETE_CMD 15 | ERR_BAD_BYTES 16 | ) 17 | 18 | /* A map to translate errors that may arise to the name of the stat that 19 | * should be reported back when the error occurs. An entry should be added 20 | * for all non-none errors that can be returned. */ 21 | var ERR_TO_STAT = map[int]string{ 22 | ERR_NO_CMD: "no_cmd", 23 | ERR_INVALID_CMD: "invalid_cmd", 24 | ERR_TRUNCATED: "truncated", 25 | ERR_INCOMPLETE_CMD: "incomplete_cmd", 26 | ERR_BAD_BYTES: "bad_bytes", 27 | } 28 | 29 | // processSingleKeyNoData processes a "get", "incr", or "decr" command, all 30 | // of which only allow for a single key to be passed and have no value field. 31 | // 32 | // On the wire, "get" looks like: 33 | // 34 | // get key\r\n 35 | // 36 | // And "incr" and "decr" look like: 37 | // 38 | // cmd key value [noreply]\r\n 39 | // 40 | // Where "noreply" is an optional field that indicates whether the server 41 | // should return a response. 42 | func processSingleKeyNoData(first_line string, remainder []byte) (keys []string, processed_remainder []byte, cmd_err int) { 43 | 44 | // Get the key 45 | // ... the command should at least consist of "cmd foo", where "foo" is the key 46 | split_data := strings.Split(first_line, " ") 47 | if len(split_data) <= 1 { 48 | return []string{}, remainder, ERR_INCOMPLETE_CMD 49 | } 50 | key := split_data[1] 51 | if key == "" { 52 | return []string{}, remainder, ERR_INCOMPLETE_CMD 53 | } 54 | 55 | // Return parsed data 56 | return []string{key}, remainder, ERR_NONE 57 | } 58 | 59 | // processSingleKeyWithData processes a "set", "add", "replace", "append", or 60 | // "prepend" command, all of which only allow for a single key and have a 61 | // corresponding value field. 62 | // 63 | // On the wire, these commands look like: 64 | // 65 | // cmd key flags exptime bytes [noreply]\r\n 66 | // \r\n 67 | // 68 | // Where "noreply" is an optional field that indicates whether the server 69 | // should return a response. 70 | func processSingleKeyWithData(first_line string, remainder []byte) (keys []string, processed_remainder []byte, cmd_err int) { 71 | 72 | // Get the key 73 | split_data := strings.Split(first_line, " ") 74 | if len(split_data) != 5 && len(split_data) != 6 { 75 | return []string{}, remainder, ERR_INCOMPLETE_CMD 76 | } 77 | key, bytes_str := split_data[1], split_data[4] 78 | 79 | // Parse length of stored value 80 | base := 10 81 | // ... the max memcached object size is 1MB, so a 32 bit int will suffice 82 | bitSize := 32 83 | bytes, err := strconv.ParseInt(bytes_str, base, bitSize) 84 | if err != nil { 85 | return []string{}, []byte{}, ERR_INVALID_CMD 86 | } 87 | 88 | // Make sure we got a full command 89 | // ... bytes + 2 to account for trailing "\r\n" 90 | next_command_idx := bytes + 2 91 | if int64(len(remainder)) < next_command_idx { 92 | return []string{}, []byte{}, ERR_TRUNCATED 93 | } 94 | 95 | // Return parsed data 96 | return []string{key}, remainder[next_command_idx:], ERR_NONE 97 | 98 | } 99 | 100 | // processMultiKeyNoData processes a "gets" command, which allows for 101 | // multiple keys and has no value field. 102 | // 103 | // On the wire, "gets" looks like: 104 | // 105 | // gets key1 key2 key3\r\n 106 | func processMultiKeyNoData(first_line string, remainder []byte) (keys []string, processed_remainder []byte, cmd_err int) { 107 | 108 | // Get the key(s) 109 | // ... the command should at least consist of "cmd foo", where "foo" is the key 110 | split_data := strings.Split(first_line, " ") 111 | if len(split_data) <= 1 { 112 | return []string{}, remainder, ERR_INCOMPLETE_CMD 113 | } 114 | keys = split_data[1:] 115 | 116 | // Return parsed data 117 | return keys, remainder, ERR_NONE 118 | } 119 | 120 | var CMD_PROCESSORS = map[string]func(first_line string, remainder []byte) (keys []string, processed_remainder []byte, cmd_err int){ 121 | "get": processSingleKeyNoData, 122 | "gets": processMultiKeyNoData, 123 | "set": processSingleKeyWithData, 124 | "add": processSingleKeyWithData, 125 | "replace": processSingleKeyWithData, 126 | "append": processSingleKeyWithData, 127 | "prepend": processSingleKeyWithData, 128 | "incr": processSingleKeyNoData, 129 | "decr": processSingleKeyNoData, 130 | } 131 | 132 | // parseCommand parses a command and list of keys the command is operating on from 133 | // a sequence of application-level data bytes. 134 | func parseCommand(app_data []byte) (cmd string, keys []string, remainder []byte, cmd_err int) { 135 | 136 | // Parse out the command 137 | space_i := bytes.IndexByte(app_data, byte(' ')) 138 | if space_i == -1 { 139 | return "", []string{}, []byte{}, ERR_NO_CMD 140 | } 141 | 142 | // Find the first newline 143 | newline_i := bytes.Index(app_data, []byte("\r\n")) 144 | if newline_i == -1 { 145 | return "", []string{}, []byte{}, ERR_TRUNCATED 146 | } 147 | 148 | // Validate command 149 | first_line := string(app_data[:newline_i]) 150 | split_data := strings.Split(first_line, " ") 151 | cmd = split_data[0] 152 | if fn, ok := CMD_PROCESSORS[cmd]; ok { 153 | keys, remainder, cmd_err = fn(first_line, app_data[newline_i+2:]) 154 | } else { 155 | return "", []string{}, []byte{}, ERR_INVALID_CMD 156 | } 157 | 158 | return cmd, keys, remainder, cmd_err 159 | } 160 | -------------------------------------------------------------------------------- /memcached_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | type ParseCommandTest struct { 9 | RawData []byte 10 | Cmd string 11 | Keys []string 12 | Remainder []byte 13 | CmdErr int 14 | } 15 | 16 | var PARSE_COMMAND_TEST_TABLE = []ParseCommandTest{ 17 | 18 | // Single Command Per Packet Tests 19 | ParseCommandTest{[]byte("set foo 0 0 3\r\nabc\r\n"), "set", []string{"foo"}, []byte{}, ERR_NONE}, 20 | ParseCommandTest{[]byte("get bar\r\n"), "get", []string{"bar"}, []byte{}, ERR_NONE}, 21 | ParseCommandTest{[]byte("\r\n"), "", []string{}, []byte{}, ERR_NO_CMD}, 22 | ParseCommandTest{[]byte("foo bar\r\n"), "", []string{}, []byte{}, ERR_INVALID_CMD}, 23 | ParseCommandTest{[]byte("get \r\n"), "get", []string{}, []byte{}, ERR_INCOMPLETE_CMD}, 24 | ParseCommandTest{[]byte("get\r\n"), "", []string{}, []byte{}, ERR_NO_CMD}, 25 | ParseCommandTest{[]byte("incr foo 1\r\n"), "incr", []string{"foo"}, []byte{}, ERR_NONE}, 26 | ParseCommandTest{[]byte("decr foo 1\r\n"), "decr", []string{"foo"}, []byte{}, ERR_NONE}, 27 | // ... test various truncation levels 28 | ParseCommandTest{[]byte("get foo"), "", []string{}, []byte{}, ERR_TRUNCATED}, 29 | ParseCommandTest{[]byte("add foo 2 44 1"), "", []string{}, []byte{}, ERR_TRUNCATED}, 30 | ParseCommandTest{[]byte("add foo 2 44 1\r"), "", []string{}, []byte{}, ERR_TRUNCATED}, 31 | ParseCommandTest{[]byte("add foo 2 44 1\r\n"), "add", []string{}, []byte{}, ERR_TRUNCATED}, 32 | ParseCommandTest{[]byte("add foo 2 44 1\r\n0"), "add", []string{}, []byte{}, ERR_TRUNCATED}, 33 | ParseCommandTest{[]byte("add foo 2 44 1\r\n0\r"), "add", []string{}, []byte{}, ERR_TRUNCATED}, 34 | ParseCommandTest{[]byte("add foo 2 44 1\r\n0\r\n"), "add", []string{"foo"}, []byte{}, ERR_NONE}, 35 | 36 | // Multiple Commands Per Packet Tests 37 | ParseCommandTest{[]byte("get foo\r\nget bar\r\n"), "get", []string{"foo"}, []byte("get bar\r\n"), ERR_NONE}, 38 | ParseCommandTest{[]byte("set foo 0 0 3\r\nabc\r\nget bar\r\n"), "set", []string{"foo"}, []byte("get bar\r\n"), ERR_NONE}, 39 | } 40 | 41 | func TestParseCommand(t *testing.T) { 42 | for test_i, test := range PARSE_COMMAND_TEST_TABLE { 43 | t.Logf(" -> parseCommand(%q)\n", test.RawData) 44 | cmd, keys, remainder, cmd_err := parseCommand(test.RawData) 45 | t.Logf(" <- %v %v %v\n", cmd, keys, cmd_err) 46 | 47 | // Verify Command 48 | if test.Cmd != cmd { 49 | t.Errorf("Test %d: expected cmd %s, got %s\n", test_i, test.Cmd, cmd) 50 | } 51 | 52 | // Verify Keys 53 | if len(test.Keys) != len(keys) { 54 | t.Errorf("Test %d: expected keys %v, got %v\n", 55 | test_i, test.Keys, keys) 56 | } else { 57 | for i := range test.Keys { 58 | if test.Keys[i] != keys[i] { 59 | t.Errorf("Test %d: expected keys %v, got %v\n", 60 | test_i, test.Keys, keys) 61 | break 62 | } 63 | } 64 | } 65 | 66 | // Verify Remainder 67 | if !bytes.Equal(test.Remainder, remainder) { 68 | t.Errorf("Test %d: expected remainder %v, got %v\n", 69 | test_i, test.Remainder, remainder) 70 | } 71 | 72 | // Verify Error 73 | if test.CmdErr != cmd_err { 74 | t.Errorf("Test %d: expected cmd err %d, got %d\n", 75 | test_i, test.CmdErr, cmd_err) 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /regexp.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "regexp" 6 | ) 7 | 8 | type RegexpKey struct { 9 | OriginalRegexp string 10 | CompiledRegexp *regexp.Regexp 11 | Name string 12 | } 13 | 14 | func NewRegexpKey(re string, name string) (regexp_key *RegexpKey, err error) { 15 | r := &RegexpKey{} 16 | compiled_regexp, err := regexp.Compile(re) 17 | if err != nil { 18 | return r, err 19 | } 20 | r.OriginalRegexp = re 21 | r.CompiledRegexp = compiled_regexp 22 | r.Name = name 23 | return r, nil 24 | } 25 | 26 | type RegexpKeys struct { 27 | regexp_keys []*RegexpKey 28 | } 29 | 30 | func NewRegexpKeys() *RegexpKeys { 31 | return &RegexpKeys{} 32 | } 33 | 34 | func (r *RegexpKeys) Add(regexp_key *RegexpKey) { 35 | r.regexp_keys = append(r.regexp_keys, regexp_key) 36 | } 37 | 38 | // Match finds the first regexp that a key matches and returns either its 39 | // associated name, or the original regex string used in its compilation 40 | func (r *RegexpKeys) Match(key string) (string, error) { 41 | for _, re := range r.regexp_keys { 42 | if re.CompiledRegexp.Match([]byte(key)) { 43 | return re.Name, nil 44 | } 45 | } 46 | return "", errors.New("Could not match key to regex.") 47 | } 48 | -------------------------------------------------------------------------------- /regexp_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | type RegexpTestItem struct { 8 | Regexp string 9 | Name string 10 | ExpectedMatches []string 11 | } 12 | 13 | var REGEXP_TEST_CASES = []RegexpTestItem{ 14 | RegexpTestItem{"^[a-f|0-9]{32}$", "MD5", []string{"aaaabbbbccccdddd1111222233334444"}}, 15 | RegexpTestItem{"^Comment_[0-9]+$", "Comment", []string{"Comment_19837"}}, 16 | RegexpTestItem{"^registered_vote_[0-9]+_t[0-9]{1}_[0-9|a-z]+$", "Registered Vote", []string{"registered_vote_9_t5_40z"}}, 17 | } 18 | 19 | func TestRegexp(t *testing.T) { 20 | for _, test_item := range REGEXP_TEST_CASES { 21 | regexp_keys := NewRegexpKeys() 22 | regexp_key, _ := NewRegexpKey(test_item.Regexp, test_item.Name) 23 | regexp_keys.Add(regexp_key) 24 | 25 | // Expected matches 26 | for _, key := range test_item.ExpectedMatches { 27 | match, _ := regexp_keys.Match(key) 28 | if match != test_item.Name { 29 | t.Errorf("Expected match %s, got %s\n", test_item.Name, match) 30 | } 31 | } 32 | } 33 | } 34 | --------------------------------------------------------------------------------