├── .gitignore
├── .travis.yml
├── Makefile
├── main_test.go
├── Vagrantfile
├── error.go
├── stats.go
├── keyfile.go
├── keys
    └── linux_keys.conf
├── zabbix_get.go
├── README.md
├── itemkey.go
└── main.go


/.gitignore:
--------------------------------------------------------------------------------
1 | .vagrant/
2 | *.tar.gz
3 | *.rpm
4 | *.deb
5 | zabbix_agent_bench
6 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: go
2 | 
3 | go:
4 |     - 1.4
5 | 
6 | install: make get-deps
7 | 
8 | script: make test
9 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | APP = zabbix_agent_bench
 2 | APPVER = 0.4.0
 3 | ARCH = $(shell uname -m)
 4 | TARBALL = $(APP)-$(APPVER).$(ARCH)
 5 | 
 6 | GO = go
 7 | GFLAGS = -x -a -v -x
 8 | RM = rm -f
 9 | FPM = fpm
10 | TAR = tar
11 | INSTALL = install -c
12 | 
13 | all: $(APP)
14 | 
15 | $(APP): main.go zabbix_get.go keyfile.go itemkey.go error.go stats.go
16 | 	$(GO) build $(GFLAGS) -o $(APP)
17 | 
18 | get-deps:
19 | 	$(GO) get -u github.com/mitchellh/colorstring
20 | 
21 | test:
22 | 	$(GO) test -x -v
23 | 
24 | install: $(APP)
25 | 	$(INSTALL) $(APP) /usr/local/bin/$(APP)
26 | 
27 | packages: tar deb rpm
28 | 
29 | tar: $(APP) README.md keys/
30 | 	mkdir $(TARBALL)
31 | 	cp -vr $(APP) README.md keys/ $(TARBALL)/
32 | 	$(TAR) -czf $(TARBALL).tar.gz $(TARBALL)/
33 | 	rm -rf $(TARBALL)/
34 | 
35 | deb: $(APP)
36 | 	$(FPM) -f -s dir -t deb -n $(APP) -v $(APPVER) $(APP)=/usr/bin/$(APP)
37 | 
38 | rpm: $(APP)
39 | 	$(FPM) -f -s dir -t rpm -n $(APP) -v $(APPVER) $(APP)=/usr/bin/$(APP)
40 | 
41 | clean:
42 | 	$(GO) clean -i -x
43 | 	$(RM) -f $(APP) $(TARBALL).tar.gz $(APP)-$(APPVER)-1.$(ARCH).rpm zabbix-agent-bench_$(APPVER)_amd64.deb
44 | 
45 | docker-run:
46 | 	docker run -it --rm -v $(PWD):/go/src/github.com/cavaliercoder/zabbix_agent_bench -w /go/src/github.com/cavaliercoder/zabbix_agent_bench golang
47 | 
48 | .PHONY: all get-deps test install packages tar deb rpm clean docker-run
49 | 
50 | 


--------------------------------------------------------------------------------
/main_test.go:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Zabbix Agent Bench (C) 2014  Ryan Armstrong <ryan@cavaliercoder.com>
 3 |  *
 4 |  * This program is free software: you can redistribute it and/or modify
 5 |  * it under the terms of the GNU General Public License as published by
 6 |  * the Free Software Foundation, either version 3 of the License, or
 7 |  * (at your option) any later version.
 8 |  *
 9 |  * This program is distributed in the hope that it will be useful,
10 |  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 |  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 |  * GNU General Public License for more details.
13 |  *
14 |  * You should have received a copy of the GNU General Public License
15 |  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 |  */
17 | package main
18 | 
19 | import (
20 | 	"os"
21 | 	"testing"
22 | )
23 | 
24 | func TestEnvVarExpansion(t *testing.T) {
25 | 	os.Setenv("VAR1", "Atom Eve")
26 | 	os.Setenv("VAR2", "Black Samson")
27 | 	os.Setenv("VAR3", "Cecil Stedman")
28 | 	os.Setenv("VAR4", "Darkwing")
29 | 
30 | 	expected := "some.key[Atom Eve,Black Samson,,,]"
31 | 	input := "  	 some.key[{%VAR1},{%VAR2},{%Var3},{%var4},{%VAR5}]"
32 | 
33 | 	key := NewItemKey(input)
34 | 
35 | 	if key.Key != expected {
36 | 		t.Errorf("Environment variable subsitution failed.\nExpected: %s\nGot:      %s", expected, key.Key)
37 | 	}
38 | }
39 | 


--------------------------------------------------------------------------------
/Vagrantfile:
--------------------------------------------------------------------------------
 1 | # -*- mode: ruby -*-
 2 | # vi: set ft=ruby :
 3 | 
 4 | script = <<script
 5 | #!/bin/bash
 6 | BULLET="==>"
 7 | 
 8 | ZBX_MAJ=2
 9 | ZBX_MIN=4
10 | ZBX_PATCH=5
11 | ZBX_REL=1
12 | ZBX_VER="${ZBX_MAJ}.${ZBX_MIN}.${ZBX_PATCH}-${ZBX_REL}"
13 | 
14 | cd /vagrant
15 | 
16 | # Install Go
17 | echo -e "${BULLET} Installing Go..."
18 | yum install -y --nogpgcheck git mercurial
19 | test -f go1.4.2.linux-amd64.tar.gz || curl -sLO https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz
20 | test -d /usr/local/go || tar -xzC /usr/local -f go1.4.2.linux-amd64.tar.gz
21 | 
22 | cat > /etc/profile.d/go.sh <<EOL
23 | export GOROOT=/usr/local/go
24 | export PATH=\\$PATH:\\$GOROOT/bin
25 | EOL
26 | . /etc/profile.d/go.sh
27 | 
28 | mkdir -p /home/vagrant/gocode/{src,pkg,bin}
29 | mkdir -p /home/vagrant/gocode/src/github.com/cavaliercoder
30 | ln -s /vagrant /home/vagrant/gocode/src/github.com/cavaliercoder/zabbix_agent_bench
31 | 
32 | grep GOPATH /home/vagrant/.bashrc >/dev/null || cat >> /home/vagrant/.bashrc <<EOL
33 | export GOPATH=\\$HOME/gocode
34 | export PATH=\\$PATH:\\$HOME/gocode/bin
35 | EOL
36 | 
37 | chown -R vagrant.vagrant /home/vagrant/gocode
38 | go get github.com/mitchellh/colorstring
39 | go version
40 | 
41 | # Install Zabbix
42 | echo -e "${BULLET} Installing Zabbix..."
43 | rpm -qa | grep zabbix-release >/dev/null || yum localinstall -y --nogpgcheck http://repo.zabbix.com/zabbix/${ZBX_MAJ}.${ZBX_MIN}/rhel/7/x86_64/zabbix-release-${ZBX_MAJ}.${ZBX_MIN}-1.el7.noarch.rpm
44 | yum install -y --nogpgcheck zabbix-agent zabbix-get
45 | chkconfig zabbix-agent on || systemctl enable zabbix-agent
46 | service zabbix-agent start || systemctl start zabbix-agent
47 | 
48 | # Install FPM
49 | echo -e "${BULLET} Installing packaging tools..."
50 | yum install -y --nogpgcheck gcc ruby-devel ruby-gems rpm-build
51 | gem install fpm
52 | 
53 | echo -e "${BULLET} All done."
54 | script
55 | 
56 | Vagrant.configure(2) do |config|
57 |   config.vm.box = "chef/centos-7.0"
58 |   config.vm.provision "shell", inline: script
59 | end
60 | 


--------------------------------------------------------------------------------
/error.go:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Zabbix Agent Bench (C) 2014  Ryan Armstrong <ryan@cavaliercoder.com>
 3 |  *
 4 |  * This program is free software: you can redistribute it and/or modify
 5 |  * it under the terms of the GNU General Public License as published by
 6 |  * the Free Software Foundation, either version 3 of the License, or
 7 |  * (at your option) any later version.
 8 |  *
 9 |  * This program is distributed in the hope that it will be useful,
10 |  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 |  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 |  * GNU General Public License for more details.
13 |  *
14 |  * You should have received a copy of the GNU General Public License
15 |  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 |  */
17 | package main
18 | 
19 | import (
20 | 	"bytes"
21 | 	"fmt"
22 | 	"github.com/mitchellh/colorstring"
23 | 	"os"
24 | )
25 | 
26 | type Error struct {
27 | 	Message    string
28 | 	InnerError error
29 | }
30 | 
31 | func NewError(err error, format string, a ...interface{}) error {
32 | 	return &Error{
33 | 		Message:    fmt.Sprintf(format, a...),
34 | 		InnerError: err,
35 | 	}
36 | }
37 | 
38 | func (c *Error) Error() string {
39 | 	buffer := bytes.NewBufferString(c.Message)
40 | 
41 | 	indent := 0
42 | 	next := c.InnerError
43 | 	for next != nil {
44 | 		buffer.WriteString("\n")
45 | 
46 | 		indent++
47 | 		for i := 0; i < indent; i++ {
48 | 			buffer.WriteString(" ")
49 | 		}
50 | 
51 | 		buffer.WriteString("-> ")
52 | 
53 | 		if nerr, ok := next.(*Error); ok {
54 | 			buffer.WriteString(nerr.Message)
55 | 			next = nerr.InnerError
56 | 		} else {
57 | 			buffer.WriteString(next.Error())
58 | 			next = nil
59 | 		}
60 | 	}
61 | 
62 | 	return buffer.String()
63 | }
64 | 
65 | func PrintError(err error) {
66 | 	colorstring.Fprintf(os.Stderr, "[red]Error:[default] %s\n", err.Error())
67 | }
68 | 
69 | func PanicOn(err error, format string, a ...interface{}) {
70 | 	if err != nil {
71 | 		PrintError(NewError(err, format, a...))
72 | 		os.Exit(1)
73 | 	}
74 | }
75 | 


--------------------------------------------------------------------------------
/stats.go:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Zabbix Agent Bench (C) 2014  Ryan Armstrong <ryan@cavaliercoder.com>
 3 |  *
 4 |  * This program is free software: you can redistribute it and/or modify
 5 |  * it under the terms of the GNU General Public License as published by
 6 |  * the Free Software Foundation, either version 3 of the License, or
 7 |  * (at your option) any later version.
 8 |  *
 9 |  * This program is distributed in the hope that it will be useful,
10 |  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 |  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 |  * GNU General Public License for more details.
13 |  *
14 |  * You should have received a copy of the GNU General Public License
15 |  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 |  */
17 | package main
18 | 
19 | import (
20 | 	"time"
21 | )
22 | 
23 | type KeyStats struct {
24 | 	Success      int64
25 | 	NotSupported int64
26 | 	Error        int64
27 | }
28 | 
29 | // ThreadStats represents the sum statistics for all item keys gathered from a
30 | // Zabbix agent by a single goroutine.
31 | type ThreadStats struct {
32 | 	Duration          time.Duration
33 | 	Iterations        int64
34 | 	TotalValues       int64
35 | 	UnsupportedValues int64
36 | 	ErrorCount        int64
37 | 	KeyStats          map[string]KeyStats
38 | }
39 | 
40 | func NewThreadStats() *ThreadStats {
41 | 	return &ThreadStats{
42 | 		KeyStats: make(map[string]KeyStats, 0),
43 | 	}
44 | }
45 | 
46 | // Add adds all stats from the specified ThreadStats struct to this ThreadStats
47 | // struct
48 | func (c *ThreadStats) Add(stats *ThreadStats) {
49 | 	c.Iterations += stats.Iterations
50 | 	c.TotalValues += stats.TotalValues
51 | 	c.UnsupportedValues += stats.UnsupportedValues
52 | 	c.ErrorCount += stats.ErrorCount
53 | 
54 | 	// add stats for each key
55 | 	for key, keyStats := range stats.KeyStats {
56 | 		tKeyStats := c.KeyStats[key]
57 | 
58 | 		tKeyStats.Success += keyStats.Success
59 | 		tKeyStats.NotSupported += keyStats.NotSupported
60 | 		tKeyStats.Error += keyStats.Error
61 | 
62 | 		c.KeyStats[key] = tKeyStats
63 | 	}
64 | }
65 | 


--------------------------------------------------------------------------------
/keyfile.go:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Zabbix Agent Bench (C) 2014  Ryan Armstrong <ryan@cavaliercoder.com>
 3 |  *
 4 |  * This program is free software: you can redistribute it and/or modify
 5 |  * it under the terms of the GNU General Public License as published by
 6 |  * the Free Software Foundation, either version 3 of the License, or
 7 |  * (at your option) any later version.
 8 |  *
 9 |  * This program is distributed in the hope that it will be useful,
10 |  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 |  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12 |  * GNU General Public License for more details.
13 |  *
14 |  * You should have received a copy of the GNU General Public License
15 |  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
16 |  */
17 | package main
18 | 
19 | import (
20 | 	"bufio"
21 | 	"os"
22 | 	"regexp"
23 | )
24 | 
25 | type KeyFile struct {
26 | 	Path string
27 | 	Keys ItemKeys
28 | }
29 | 
30 | var commentPattern = regexp.MustCompile(`^\s*(#.*)?$`)
31 | 
32 | // NewKeyFile loads Zabbix agent keys from a plain text file
33 | func NewKeyFile(path string) (*KeyFile, error) {
34 | 
35 | 	// Open key file
36 | 	dprintf("Loading keys from file: %s\n", path)
37 | 	file, err := os.Open(path)
38 | 	if err != nil {
39 | 		return nil, err
40 | 	}
41 | 	defer file.Close()
42 | 
43 | 	var keyfile = &KeyFile{
44 | 		Path: path,
45 | 		Keys: make(ItemKeys, 0),
46 | 	}
47 | 
48 | 	var (
49 | 		lastKey   *ItemKey
50 | 		parentKey *ItemKey
51 | 	)
52 | 
53 | 	// Read one key per line
54 | 	buf := bufio.NewScanner(file)
55 | 	for buf.Scan() {
56 | 		line := buf.Text()
57 | 
58 | 		// Ignore blanks lines and comments
59 | 		if !commentPattern.MatchString(line) {
60 | 			newKey := NewItemKey(line)
61 | 
62 | 			// is this a child prototype item?
63 | 			if indentPattern.MatchString(line) {
64 | 				dprintf("Added key prototype: %s\n", newKey.Key)
65 | 
66 | 				// Make the parent a Discovery Rule if not already
67 | 				newKey.IsPrototype = true
68 | 				if parentKey == nil {
69 | 					parentKey = lastKey
70 | 					parentKey.IsDiscoveryRule = true
71 | 				}
72 | 
73 | 				// Append to parent
74 | 				parentKey.Prototypes = append(parentKey.Prototypes, newKey)
75 | 			} else {
76 | 				// This is a normal key
77 | 				dprintf("Added key: %s\n", newKey.Key)
78 | 				parentKey = nil
79 | 				keyfile.Keys = append(keyfile.Keys, newKey)
80 | 			}
81 | 
82 | 			lastKey = newKey
83 | 		}
84 | 	}
85 | 
86 | 	dprintf("Finished loading key file\n")
87 | 	return keyfile, nil
88 | }
89 | 


--------------------------------------------------------------------------------
/keys/linux_keys.conf:
--------------------------------------------------------------------------------
  1 | # Zabbix Linux Agent (v2.2.6) keys
  2 | agent.hostname
  3 | agent.ping
  4 | agent.version
  5 | kernel.maxfiles
  6 | kernel.maxproc
  7 | 
  8 | net.if.discovery
  9 |     net.if.in[{#IFNAME}]
 10 |     net.if.in[{#IFNAME},bytes]
 11 |     net.if.in[{#IFNAME},packets]
 12 |     net.if.in[{#IFNAME},errors]
 13 |     net.if.in[{#IFNAME},dropped]
 14 |     net.if.out[{#IFNAME}]
 15 |     net.if.out[{#IFNAME},bytes]
 16 |     net.if.out[{#IFNAME},packets]
 17 |     net.if.out[{#IFNAME},errors]
 18 |     net.if.out[{#IFNAME},dropped]
 19 |     net.if.total[{#IFNAME}]
 20 |     net.if.total[{#IFNAME},bytes]
 21 |     net.if.total[{#IFNAME},packets]
 22 |     net.if.total[{#IFNAME},errors]
 23 |     net.if.total[{#IFNAME},dropped]
 24 |     net.if.collisions[{#IFNAME}]
 25 | 
 26 | vfs.fs.discovery
 27 |     vfs.fs.inode[{#FSNAME},total]
 28 |     vfs.fs.inode[{#FSNAME},free]
 29 |     vfs.fs.inode[{#FSNAME},used]
 30 |     vfs.fs.inode[{#FSNAME},pfree]
 31 |     vfs.fs.inode[{#FSNAME},pused]
 32 |     vfs.fs.size[{#FSNAME},total]
 33 |     vfs.fs.size[{#FSNAME},free]
 34 |     vfs.fs.size[{#FSNAME},used]
 35 |     vfs.fs.size[{#FSNAME},pfree]
 36 |     vfs.fs.size[{#FSNAME},pused]
 37 | 
 38 | net.tcp.listen[22]
 39 | net.tcp.service[ssh]
 40 | net.tcp.service.perf[ssh]
 41 | net.udp.listen[111]
 42 | 
 43 | proc.mem[zabbix_agentd]
 44 | proc.mem[,root,avg]
 45 | proc.mem[,root,min]
 46 | proc.mem[,root,max]
 47 | proc.mem[,root,sum]
 48 | 
 49 | proc.num[zabbix_agentd]
 50 | proc.num[,root,all]
 51 | proc.num[,root,run]
 52 | proc.num[,root,sleep]
 53 | proc.num[,root,zomb]
 54 | 
 55 | # sensor[device,sensor,<mode>] having trouble getting this key to work on CentOS 7.0
 56 | 
 57 | system.boottime
 58 | system.hostname
 59 | 
 60 | system.hw.chassis[full]
 61 | system.hw.chassis[model]
 62 | system.hw.chassis[serial]
 63 | system.hw.chassis[type]
 64 | system.hw.chassis[vendor]
 65 | 
 66 | system.hw.macaddr[,full]
 67 | system.hw.macaddr[,short]
 68 | 
 69 | system.localtime[utc]
 70 | system.localtime[local]
 71 | 
 72 | system.cpu.intr
 73 | system.cpu.load
 74 | system.cpu.load[all,avg1]
 75 | system.cpu.load[all,avg5]
 76 | system.cpu.load[all,avg15]
 77 | system.cpu.num
 78 | system.cpu.num[online]
 79 | system.cpu.num[max]
 80 | system.cpu.switches
 81 | system.uname
 82 | system.uptime
 83 | system.users.num
 84 | system.sw.arch
 85 | system.sw.os[full]
 86 | system.sw.os[short]
 87 | system.sw.os[name]
 88 | 
 89 | vm.memory.size[total]
 90 | vm.memory.size[cached]
 91 | vm.memory.size[used]
 92 | vm.memory.size[pused]
 93 | vm.memory.size[available]
 94 | vm.memory.size[pavailable]
 95 | 
 96 | #
 97 | # The following keys require a fork:
 98 | #
 99 | # system.run[/usr/bin/ls,wait]
100 | # system.run[/usr/bin/ls,nowait]
101 | # system.hw.devices[pci]
102 | # system.hw.devices[usb]
103 | 
104 | #
105 | # The following keys introduce network latency when stress testing:
106 | #
107 | # net.dns[,google.com]
108 | # net.dns.record[,google.com,A]
109 | 


--------------------------------------------------------------------------------
/zabbix_get.go:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Zabbix Agent Bench (C) 2014  Ryan Armstrong <ryan@cavaliercoder.com>
  3 |  *
  4 |  * This program is free software: you can redistribute it and/or modify
  5 |  * it under the terms of the GNU General Public License as published by
  6 |  * the Free Software Foundation, either version 3 of the License, or
  7 |  * (at your option) any later version.
  8 |  *
  9 |  * This program is distributed in the hope that it will be useful,
 10 |  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 11 |  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 12 |  * GNU General Public License for more details.
 13 |  *
 14 |  * You should have received a copy of the GNU General Public License
 15 |  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 16 |  */
 17 | 
 18 | /*
 19 |  * This file courtesy: https://github.com/fujiwara/go-zabbix-get
 20 |  */
 21 | 
 22 | package main
 23 | 
 24 | import (
 25 | 	"bytes"
 26 | 	"encoding/binary"
 27 | 	"fmt"
 28 | 	"io"
 29 | 	"net"
 30 | 	"time"
 31 | )
 32 | 
 33 | const (
 34 | 	AgentDefaultPort = 10050
 35 | 	HeaderString     = "ZBXD"
 36 | 	HeaderLength     = len(HeaderString)
 37 | 	HeaderVersion    = uint8(1)
 38 | 	DataLengthOffset = int64(HeaderLength + 1)
 39 | 	DataLengthSize   = int64(8)
 40 | 	DataOffset       = int64(DataLengthOffset + DataLengthSize)
 41 | 	ErrorMessage     = "ZBX_NOTSUPPORTED"
 42 | )
 43 | 
 44 | var (
 45 | 	ErrorMessageBytes = []byte(ErrorMessage)
 46 | 	Terminator        = []byte("\n")
 47 | 	HeaderBytes       = []byte(HeaderString)
 48 | )
 49 | 
 50 | func Get(addr string, key string, timeout time.Duration) (value string, err error) {
 51 | 	// Append port specifier to socket address
 52 | 	_, _, err = net.SplitHostPort(addr)
 53 | 	if err != nil {
 54 | 		addr = fmt.Sprintf("%s:%d", addr, AgentDefaultPort)
 55 | 	}
 56 | 
 57 | 	// Connect via TCP
 58 | 	conn, err := net.DialTimeout("tcp", addr, timeout)
 59 | 	if err != nil {
 60 | 		return
 61 | 	}
 62 | 	defer conn.Close()
 63 | 	conn.SetDeadline(time.Now().Add(timeout))
 64 | 
 65 | 	// Build the request
 66 | 	buf := new(bytes.Buffer)
 67 | 	buf.Write(HeaderBytes)
 68 | 	binary.Write(buf, binary.LittleEndian, HeaderVersion)
 69 | 	binary.Write(buf, binary.LittleEndian, int64(len([]byte(key))))
 70 | 	buf.Write([]byte(key))
 71 | 
 72 | 	// Send the request
 73 | 	_, err = conn.Write(buf.Bytes())
 74 | 	if err != nil {
 75 | 		return
 76 | 	}
 77 | 
 78 | 	// read header "ZBXD\x01"
 79 | 	head := make([]byte, DataLengthOffset)
 80 | 	_, err = conn.Read(head)
 81 | 	if err != nil {
 82 | 		return
 83 | 	}
 84 | 
 85 | 	val, err := parseBinary(conn)
 86 | 
 87 | 	return string(val), err
 88 | }
 89 | 
 90 | func parseBinary(conn io.Reader) (rdata []byte, err error) {
 91 | 	// read data length
 92 | 	var dataLength int64
 93 | 	err = binary.Read(conn, binary.LittleEndian, &dataLength)
 94 | 	if err != nil {
 95 | 		return
 96 | 	}
 97 | 
 98 | 	// read data body
 99 | 	buf := make([]byte, 1024)
100 | 	data := new(bytes.Buffer)
101 | 	total := 0
102 | 	size := 0
103 | 	for total < int(dataLength) {
104 | 		size, err = conn.Read(buf)
105 | 		if err != nil {
106 | 			return
107 | 		}
108 | 		if size == 0 {
109 | 			break
110 | 		}
111 | 		total = total + size
112 | 		data.Write(buf[0:size])
113 | 	}
114 | 	rdata = data.Bytes()
115 | 	return
116 | }
117 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Zabbix Agent Bench [![Build Status](https://travis-ci.org/cavaliercoder/zabbix_agent_bench.svg?branch=master)](https://travis-ci.org/cavaliercoder/zabbix_agent_bench) [![Download zabbix_agent_bench](https://img.shields.io/sourceforge/dm/zabbixagentbench.svg)](https://sourceforge.net/projects/zabbixagentbench/files/)
  2 | 
  3 | A multithreaded Zabbix agent benchmarking tool with support for custom keys and
  4 | discovery item prototypes.
  5 | 
  6 | This tool is useful for developing custom Zabbix agent items and quickly
  7 | identifying memory or file handle leaks, concurrency problems such as race
  8 | conditions and other performance issues.
  9 | 
 10 | ## Usage
 11 | 
 12 |     $ zabbix_agent_bench --help
 13 |     Usage of ./zabbix_agent_bench:
 14 |       -debug
 15 |           print program debug messages
 16 |       -delay int
 17 |           delay between queries on each thread in milliseconds
 18 |       -host string
 19 |           remote Zabbix agent host (default "localhost")
 20 |       -iterations int
 21 |           maximum test iterations of each key
 22 |       -key string
 23 |           benchmark a single agent item key
 24 |       -keys string
 25 |           read keys from file path
 26 |       -offset int
 27 |           delay start of each thread in milliseconds
 28 |       -port int
 29 |           remote Zabbix agent TCP port (default 10050)
 30 |       -strict
 31 |           exit code to include tally of unsupported items
 32 |       -threads int
 33 |           number of test threads (default 4)
 34 |       -timelimit int
 35 |           time limit in seconds
 36 |       -timeout int
 37 |           timeout in milliseconds for each zabbix_get request (default 3000)
 38 |       -verbose
 39 |           print more output
 40 |       -version
 41 |           print version
 42 | 
 43 | Test a single key until cancelled with `Ctrl-C`:
 44 | 
 45 |     $ zabbix_agent_bench -key agent.version
 46 | 
 47 | Test a list of keys (including discovery rules and prototypes):
 48 | 
 49 |     $ zabbix_agent_bench -keys linux_keys.conf
 50 | 
 51 | Simple unit-test style check of a list of keys:
 52 | 
 53 |     $ zabbix_agent_bench -keys linux_keys.conf -iterations 1 -strict
 54 | 
 55 | 
 56 | ## Key files
 57 | 
 58 | You can test multiple keys by creating a text file with one key per line. You
 59 | may then pass this file to the `-keys` argument.
 60 | 
 61 | See the [libzbxpgsql](https://github.com/cavaliercoder/libzbxpgsql/blob/master/fixtures/postgresql-9.2.keys)
 62 | project for a substantial key file example.
 63 | 
 64 | To create a discovery rule, you may specify item prototypes immediately
 65 | following an item definition, simply by prepending the prototype key with a tab
 66 | or space.
 67 | 
 68 | E.g.
 69 | 
 70 |     vfs.fs.discovery
 71 |         vfs.fs.size[{#FSNAME},total]
 72 |         vfs.fs.size[{#FSNAME},free]
 73 |         vfs.fs.size[{#FSNAME},used]
 74 |         vfs.fs.size[{#FSNAME},pfree]
 75 |         vfs.fs.size[{#FSNAME},pused]
 76 | 
 77 | Whitespace and lines prefixed with `#` are ignored as comments.
 78 | 
 79 | To substitute key names with runtime environment variables, you can use the
 80 | form `{%VARNAME}` where `VARNAME` is the case-sensitive name of en environment
 81 | variable.
 82 | 
 83 | For example, if you set environment variable `TCPPORT` to `22`, then the
 84 | following key file entry:
 85 | 
 86 |     net.tcp.listen[{%TCPPORT}}]
 87 | 
 88 | will be expanded to:
 89 | 
 90 |     net.tcp.listen[22]
 91 | 
 92 | This enables a key file to be reused with ease when testing against various
 93 | target environments.
 94 | 
 95 | E.g.
 96 | 
 97 |     $ TCPPORT=22 zabbix_agent_bench -keys linux_keys.conf
 98 | 
 99 | Variables which are not set in the environment are replaced with a zero length
100 | string. In the example above, if `TCPPORT` was not set, the key would become:
101 | 
102 |     net.tcp.listen[]
103 | 
104 | 
105 | ## Installation
106 | 
107 | Pre-compiled binaries and packages are available for
108 | [download on SourceForge](https://sourceforge.net/projects/zabbixagentbench/files/).
109 | 
110 | Alternatively, you can build the project yourself in Go. Once you have a
111 | working [installation of Go](https://golang.org/doc/install), simply run:
112 | 
113 |     $ go get github.com/cavaliercoder/zabbix_agent_bench
114 | 
115 | 
116 | ## License
117 | 
118 | Zabbix Agent Bench Copyright (C) 2014 Ryan Armstrong (ryan@cavaliercoder.com)
119 | 
120 | This program is free software: you can redistribute it and/or modify it under
121 | the terms of the GNU General Public License as published by the Free Software
122 | Foundation, either version 3 of the License, or (at your option) any later
123 | version.
124 | 
125 | This program is distributed in the hope that it will be useful, but WITHOUT ANY
126 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
127 | PARTICULAR PURPOSE. See the GNU General Public License for more details.
128 | 
129 | You should have received a copy of the GNU General Public License along with
130 | this program. If not, see http://www.gnu.org/licenses/.
131 | 


--------------------------------------------------------------------------------
/itemkey.go:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Zabbix Agent Bench (C) 2014  Ryan Armstrong <ryan@cavaliercoder.com>
  3 |  *
  4 |  * This program is free software: you can redistribute it and/or modify
  5 |  * it under the terms of the GNU General Public License as published by
  6 |  * the Free Software Foundation, either version 3 of the License, or
  7 |  * (at your option) any later version.
  8 |  *
  9 |  * This program is distributed in the hope that it will be useful,
 10 |  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 11 |  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 12 |  * GNU General Public License for more details.
 13 |  *
 14 |  * You should have received a copy of the GNU General Public License
 15 |  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 16 |  */
 17 | package main
 18 | 
 19 | import (
 20 | 	"encoding/json"
 21 | 	"os"
 22 | 	"regexp"
 23 | 	"sort"
 24 | 	"strings"
 25 | 	"time"
 26 | )
 27 | 
 28 | // An ItemKey is a single Zabbix agent item check key
 29 | type ItemKey struct {
 30 | 	Key             string
 31 | 	IsDiscoveryRule bool
 32 | 	IsPrototype     bool
 33 | 	Prototypes      ItemKeys
 34 | }
 35 | 
 36 | // ItemKeys is an array of pointers to ItemKey structs
 37 | type ItemKeys []*ItemKey
 38 | 
 39 | // DiscoveryData is a key/val array of values returned embedded in a discovery
 40 | // rule response
 41 | type DiscoveryData []map[string]string
 42 | 
 43 | // DiscoveryResponse is the JSON packet returned by the Zabbix agent for
 44 | // discovery rules
 45 | type DiscoveryResponse struct {
 46 | 	Data DiscoveryData
 47 | }
 48 | 
 49 | var (
 50 | 	envVarPattern = regexp.MustCompile(`\{%.*?\}`)
 51 | 	indentPattern = regexp.MustCompile(`^\s+`)
 52 | )
 53 | 
 54 | // NewItemKey returns a pointer to a new instance of an ItemKey with
 55 | // environment variables in the key name expanded.
 56 | func NewItemKey(key string) *ItemKey {
 57 | 	return &ItemKey{
 58 | 		Key:             ParseItemKey(key),
 59 | 		IsDiscoveryRule: false,
 60 | 		IsPrototype:     false,
 61 | 		Prototypes:      make(ItemKeys, 0),
 62 | 	}
 63 | }
 64 | 
 65 | // LongestKeyName returns the length in characters of the longest key name
 66 | // in an array of keys.
 67 | // Used for formatting output.
 68 | func (c ItemKeys) LongestKeyName() int {
 69 | 	longestKeyName := 0
 70 | 	for _, key := range c {
 71 | 		if len(key.Key) > longestKeyName {
 72 | 			longestKeyName = len(key.Key)
 73 | 		}
 74 | 	}
 75 | 
 76 | 	return longestKeyName
 77 | }
 78 | 
 79 | // SortedKeyNames returns the name of all keys in the this key array sorted
 80 | // alphanumerically.
 81 | func (c ItemKeys) SortedKeyNames() []string {
 82 | 	keyNames := []string{}
 83 | 	for _, key := range c {
 84 | 		keyNames = append(keyNames, key.Key)
 85 | 	}
 86 | 	sort.Strings(keyNames)
 87 | 
 88 | 	return keyNames
 89 | }
 90 | 
 91 | // ParseItemKey subsitutes any variables in a key name for runtime
 92 | // environment variables and trims any whitespace at the beginning of the key.
 93 | //
 94 | // Variables in the key name take the form '{%VARNAME}' and are replaced with
 95 | // the matching environment variable value (e.g. 'VARNAME').
 96 | //
 97 | // Variables with no value set in the runtime environment are replaced with a
 98 | // zero length string.
 99 | func ParseItemKey(key string) string {
100 | 	// Strip out indentation
101 | 	key = indentPattern.ReplaceAllString(key, "")
102 | 
103 | 	// replace environment variables
104 | 	vars := envVarPattern.FindAllString(key, -1)
105 | 	for _, v := range vars {
106 | 		ev := v[2 : len(v)-1]
107 | 		key = strings.Replace(key, v, os.Getenv(ev), -1)
108 | 	}
109 | 
110 | 	return key
111 | }
112 | 
113 | // Discover sends a 'get' request to a Zabbix agent and expand the key's
114 | // discovery prototypes into new standard keys using the response from the
115 | // Zabbix agent
116 | func (c *ItemKey) Discover(host string, timeout time.Duration) (ItemKeys, error) {
117 | 	if !c.IsDiscoveryRule {
118 | 		return nil, NewError(nil, "Item is not a discovery rule: %s", c.Key)
119 | 	}
120 | 
121 | 	// get discovery items to expand prototypes
122 | 	dprintf("Executing discovery rule: %s\n", c.Key)
123 | 	val, err := Get(host, c.Key, timeout)
124 | 	if err != nil {
125 | 		return nil, NewError(err, "Failed to get discovery data for item: %s", c.Key)
126 | 	}
127 | 
128 | 	// check if result is unsupported
129 | 	if strings.HasPrefix(val, ZBX_NOTSUPPORTED) {
130 | 		return nil, NewError(nil, "Discovery rule unsupported for item: %s", c.Key)
131 | 	}
132 | 
133 | 	// bind JSON discovery data
134 | 	response := DiscoveryResponse{}
135 | 	err = json.Unmarshal([]byte(val), &response)
136 | 	if err != nil {
137 | 		return nil, NewError(err, "Failed to parse discovery json data for item: %s\n%s", c.Key, val)
138 | 	}
139 | 
140 | 	// Parse each discovered instance
141 | 	keys := ItemKeys{}
142 | 	for _, instance := range response.Data {
143 | 
144 | 		// Create prototypes
145 | 		for _, proto := range c.Prototypes {
146 | 
147 | 			// Expand macros
148 | 			s := proto.Key
149 | 			for macro, val := range instance {
150 | 				dprintf("Substituting macro '%s' with value '%s'\n", macro, val)
151 | 				s = strings.Replace(s, macro, val, -1)
152 | 			}
153 | 
154 | 			// Item discovered item
155 | 			n := NewItemKey(s)
156 | 			n.IsPrototype = true
157 | 
158 | 			keys = append(keys, n)
159 | 
160 | 			dprintf("Added discovered key: %s\n", n.Key)
161 | 		}
162 | 	}
163 | 
164 | 	return keys, nil
165 | }
166 | 
167 | // Expand executes a discovery on all discovery rules in a list of keys and
168 | // appends the expanded prototypes to the returned array
169 | func (c ItemKeys) Expand(host string, timeout time.Duration) (ItemKeys, error) {
170 | 	keys := c
171 | 	for _, key := range c {
172 | 		if key.IsDiscoveryRule {
173 | 			discoveredKeys, err := key.Discover(host, timeout)
174 | 			if err != nil {
175 | 				return nil, NewError(err, "Failed to expand prototypes for discovery rule: %s", key.Key)
176 | 			}
177 | 
178 | 			keys = append(keys, discoveredKeys...)
179 | 		}
180 | 	}
181 | 
182 | 	return keys, nil
183 | }
184 | 


--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Zabbix Agent Bench (C) 2014  Ryan Armstrong <ryan@cavaliercoder.com>
  3 |  *
  4 |  * This program is free software: you can redistribute it and/or modify
  5 |  * it under the terms of the GNU General Public License as published by
  6 |  * the Free Software Foundation, either version 3 of the License, or
  7 |  * (at your option) any later version.
  8 |  *
  9 |  * This program is distributed in the hope that it will be useful,
 10 |  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 11 |  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 12 |  * GNU General Public License for more details.
 13 |  *
 14 |  * You should have received a copy of the GNU General Public License
 15 |  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 16 |  */
 17 | package main
 18 | 
 19 | import (
 20 | 	"flag"
 21 | 	"fmt"
 22 | 	"github.com/mitchellh/colorstring"
 23 | 	"os"
 24 | 	"os/signal"
 25 | 	"runtime"
 26 | 	"strings"
 27 | 	"time"
 28 | )
 29 | 
 30 | const (
 31 | 	APP         = "zabbix_agent_bench"
 32 | 	APP_VERSION = "0.4.0"
 33 | 	APP_AUTHOR  = "Ryan Armstrong <ryan@cavaliercoder.com>"
 34 | 
 35 | 	ZBX_NOTSUPPORTED = "ZBX_NOTSUPPORTED"
 36 | )
 37 | 
 38 | // command args
 39 | var (
 40 | 	debug          bool
 41 | 	exitErrorCount bool
 42 | 	host           string
 43 | 	iterationLimit int
 44 | 	key            string
 45 | 	keyFilePath    string
 46 | 	port           int
 47 | 	delayMsArg     int
 48 | 	staggerMsArg   int
 49 | 	threadCount    int
 50 | 	timeLimitArg   int
 51 | 	timeoutMsArg   int
 52 | 	verbose        bool
 53 | 	version        bool
 54 | )
 55 | 
 56 | var (
 57 | 	timeout       time.Duration
 58 | 	delayDuration time.Duration
 59 | )
 60 | 
 61 | // flag to signal all threads to stop gracefully
 62 | var stop = false
 63 | 
 64 | func main() {
 65 | 
 66 | 	// Configure from command line
 67 | 	flag.BoolVar(&version, "version", false, "print version")
 68 | 	flag.StringVar(&host, "host", "localhost", "remote Zabbix agent host")
 69 | 	flag.IntVar(&port, "port", 10050, "remote Zabbix agent TCP port")
 70 | 	flag.IntVar(&timeoutMsArg, "timeout", 3000, "timeout in milliseconds for each zabbix_get request")
 71 | 	flag.IntVar(&delayMsArg, "delay", 0, "delay between queries on each thread in milliseconds")
 72 | 	flag.IntVar(&staggerMsArg, "offset", 0, "delay start of each thread in milliseconds")
 73 | 	flag.IntVar(&threadCount, "threads", runtime.NumCPU(), "number of test threads")
 74 | 	flag.IntVar(&timeLimitArg, "timelimit", 0, "time limit in seconds")
 75 | 	flag.IntVar(&iterationLimit, "iterations", 0, "maximum test iterations of each key")
 76 | 	flag.StringVar(&keyFilePath, "keys", "", "read keys from file path")
 77 | 	flag.StringVar(&key, "key", "", "benchmark a single agent item key")
 78 | 	flag.BoolVar(&exitErrorCount, "strict", false, "exit code to include tally of unsupported items")
 79 | 	flag.BoolVar(&verbose, "verbose", false, "print more output")
 80 | 	flag.BoolVar(&debug, "debug", false, "print program debug messages")
 81 | 	flag.Parse()
 82 | 
 83 | 	timeout = time.Duration(timeoutMsArg) * time.Millisecond
 84 | 	stagger := time.Duration(staggerMsArg) * time.Millisecond
 85 | 	delayDuration = time.Duration(delayMsArg) * time.Millisecond
 86 | 	timeLimit := time.Duration(timeLimitArg) * time.Second
 87 | 
 88 | 	// print version and exit
 89 | 	if version {
 90 | 		fmt.Printf("%s v%s\n", APP, APP_VERSION)
 91 | 		os.Exit(0)
 92 | 	}
 93 | 
 94 | 	// Bind threads to each core
 95 | 	runtime.GOMAXPROCS(runtime.NumCPU())
 96 | 
 97 | 	// Create a list of keys for processing
 98 | 	queuedKeys := ItemKeys{}
 99 | 
100 | 	// user specified a single key
101 | 	if key != "" {
102 | 		queuedKeys = append(queuedKeys, NewItemKey(key))
103 | 	}
104 | 
105 | 	// load item keys from text file
106 | 	if keyFilePath != "" {
107 | 		keyFile, err := NewKeyFile(keyFilePath)
108 | 		PanicOn(err, "Failed to open key file")
109 | 
110 | 		// expand discovery item prototypes by doing an actual agent discovery
111 | 		queuedKeys, err = keyFile.Keys.Expand(host, timeout)
112 | 		PanicOn(err, "Failed to expand discovery items")
113 | 	}
114 | 
115 | 	// Make sure we have work to do
116 | 	if 0 == len(queuedKeys) {
117 | 		fmt.Fprintf(os.Stderr, "No agent item keys specified for testing\n")
118 | 		os.Exit(1)
119 | 	}
120 | 
121 | 	// TODO: deduplicate the key list
122 | 
123 | 	// start producer thread
124 | 	fmt.Printf("Testing %d keys with %d threads (press Ctrl-C to cancel)...\n", len(queuedKeys), threadCount)
125 | 	HandleSignals()
126 | 	statsChan := make(chan *ThreadStats)
127 | 	producer := StartProducer(queuedKeys, statsChan)
128 | 
129 | 	// set time limit if set
130 | 	if 0 < timeLimit {
131 | 		timer := time.NewTimer(timeLimit)
132 | 		go func() {
133 | 			<-timer.C
134 | 			stop = true
135 | 		}()
136 | 	}
137 | 	start := time.Now()
138 | 
139 | 	// fan out consumer threads to start work
140 | 	for i := 0; !stop && i < threadCount; i++ {
141 | 		// Stagger thread start
142 | 		time.Sleep(stagger)
143 | 
144 | 		dprintf("Starting thread %d...\n", i+1)
145 | 		go StartConsumer(producer, statsChan)
146 | 	}
147 | 
148 | 	// Fan in threads to gather stats
149 | 	totals := NewThreadStats()
150 | 	for i := 0; i < threadCount+1; i++ {
151 | 		threadStats := <-statsChan
152 | 		totals.Add(threadStats)
153 | 	}
154 | 
155 | 	duration := time.Now().Sub(start)
156 | 
157 | 	// Sort the key list
158 | 	keyNames := queuedKeys.SortedKeyNames()
159 | 
160 | 	// Print results per key
161 | 	longestKeyName := queuedKeys.LongestKeyName()
162 | 	for _, key := range keyNames {
163 | 		keyStats := totals.KeyStats[key]
164 | 
165 | 		// escape %'s in key name
166 | 		key = strings.Replace(key, "%", "%%", -1)
167 | 
168 | 		// show stats
169 | 		row := fmt.Sprintf("%-*s :\t%s\t%s\t%s\n", longestKeyName, key, hl(keyStats.Success, "green"), hl(keyStats.NotSupported, "yellow"), hl(keyStats.Error, "red"))
170 | 		colorstring.Printf(row)
171 | 	}
172 | 
173 | 	// Print totals
174 | 	fmt.Printf("\n=== Totals ===\n\n")
175 | 	fmt.Printf("Total values processed:\t\t%d\n", totals.TotalValues)
176 | 	fmt.Printf("Total unsupported values:\t%d\n", totals.UnsupportedValues)
177 | 	fmt.Printf("Total transport errors:\t\t%d\n", totals.ErrorCount)
178 | 	fmt.Printf("Total key list iterations:\t%d\n", totals.Iterations)
179 | 
180 | 	colorstring.Printf("\n[green]Finished![default] Processed %d values across %d threads in %s (%f NVPS)\n", totals.TotalValues, threadCount, duration.String(), (float64(totals.TotalValues) / duration.Seconds()))
181 | 
182 | 	// exit code
183 | 	if exitErrorCount {
184 | 		os.Exit(int(totals.UnsupportedValues + totals.ErrorCount))
185 | 	} else {
186 | 		os.Exit(int(totals.ErrorCount))
187 | 	}
188 | }
189 | 
190 | // HandleSignals starts a new goroutine to handle signals from the operating
191 | // system and signal other goroutine to gracefully stop.
192 | func HandleSignals() {
193 | 	c := make(chan os.Signal, 1)
194 | 	signal.Notify(c, os.Interrupt)
195 | 	go func() {
196 | 		for {
197 | 			<-c // Wait for signal
198 | 
199 | 			if stop {
200 | 				// Force exit if user sent SIGINT during cleanup
201 | 				fmt.Printf("Aborting...\n")
202 | 				os.Exit(1)
203 | 			} else {
204 | 				fmt.Printf("Caught SIGINT. Cleaning up...\n")
205 | 				stop = true
206 | 			}
207 | 		}
208 | 	}()
209 | }
210 | 
211 | // StartProducer starts a goroutine which iterates through the list of queued
212 | // agent item check keys and published them sequentially to the returned
213 | // channel until the runtime limits are reached.
214 | func StartProducer(keys ItemKeys, statsChan chan *ThreadStats) <-chan *ItemKey {
215 | 	c := make(chan *ItemKey)
216 | 	go func() {
217 | 		stats := ThreadStats{}
218 | 		for i := 0; !stop && (iterationLimit <= 0 || i < iterationLimit); i++ {
219 | 			for _, key := range keys {
220 | 				if stop {
221 | 					break
222 | 				}
223 | 
224 | 				// send key to a consumer
225 | 				c <- key
226 | 			}
227 | 
228 | 			stats.Iterations++
229 | 		}
230 | 
231 | 		close(c)
232 | 		statsChan <- &stats
233 | 	}()
234 | 
235 | 	return c
236 | }
237 | 
238 | // StartConsumer consumes ItemKeys from a producer channel, queries the Zabbix
239 | // agent for a response and submits the results to a ThreadStats channel.
240 | func StartConsumer(producer <-chan *ItemKey, statsChan chan *ThreadStats) {
241 | 	threadStats := NewThreadStats()
242 | 
243 | 	// process items as long the producer produces them
244 | 	for key := range producer {
245 | 		keyStats := threadStats.KeyStats[key.Key]
246 | 
247 | 		// Get the value from Zabbix agent
248 | 		val, err := Get(host, key.Key, timeout)
249 | 
250 | 		// tally stats
251 | 		if err != nil {
252 | 			threadStats.ErrorCount++
253 | 			keyStats.Error++
254 | 		} else {
255 | 			threadStats.TotalValues++
256 | 			if strings.HasPrefix(val, ErrorMessage) {
257 | 				threadStats.UnsupportedValues++
258 | 				keyStats.NotSupported++
259 | 			} else {
260 | 				keyStats.Success++
261 | 			}
262 | 
263 | 			// Print response
264 | 			if verbose {
265 | 				typ := "item"
266 | 				if key.IsPrototype {
267 | 					typ = "proto"
268 | 				} else if key.IsDiscoveryRule {
269 | 					typ = "disco"
270 | 				}
271 | 
272 | 				fmt.Printf("[%s] %s: %s\n", typ, key.Key, val)
273 | 			}
274 | 		}
275 | 
276 | 		threadStats.KeyStats[key.Key] = keyStats
277 | 
278 | 		// sleep
279 | 		if delayMsArg > 0 && !stop {
280 | 			time.Sleep(delayDuration)
281 | 		}
282 | 	}
283 | 
284 | 	// Push stats to collector channel
285 | 	statsChan <- threadStats
286 | }
287 | 
288 | // dprintf prints debug output if debug is enabled.
289 | func dprintf(format string, a ...interface{}) {
290 | 	if debug {
291 | 		fmt.Fprintf(os.Stderr, format, a...)
292 | 	}
293 | }
294 | 
295 | func hl(val int64, color string) string {
296 | 	if val > 0 {
297 | 		return fmt.Sprintf("[%s]%d[default]", color, val)
298 | 	} else {
299 | 		return fmt.Sprintf("%d", val)
300 | 	}
301 | }
302 | 


--------------------------------------------------------------------------------