5 |
9 |
10 |
6 |
8 | Topics
7 |
11 |
34 |
--------------------------------------------------------------------------------
/internal/protocol/errors.go:
--------------------------------------------------------------------------------
1 | package protocol
2 |
3 | type ChildErr interface {
4 | Parent() error
5 | }
6 |
7 | // ClientErr provides a way for NSQ daemons to log a human reabable
8 | // error string and return a machine readable string to the client.
9 | //
10 | // see docs/protocol.md for error codes by command
11 | type ClientErr struct {
12 | ParentErr error
13 | Code string
14 | Desc string
15 | }
16 |
17 | // Error returns the machine readable form
18 | func (e *ClientErr) Error() string {
19 | return e.Code + " " + e.Desc
20 | }
21 |
22 | // Parent returns the parent error
23 | func (e *ClientErr) Parent() error {
24 | return e.ParentErr
25 | }
26 |
27 | // NewClientErr creates a ClientErr with the supplied human and machine readable strings
28 | func NewClientErr(parent error, code string, description string) *ClientErr {
29 | return &ClientErr{parent, code, description}
30 | }
31 |
32 | type FatalClientErr struct {
33 | ParentErr error
34 | Code string
35 | Desc string
36 | }
37 |
38 | // Error returns the machine readable form
39 | func (e *FatalClientErr) Error() string {
40 | return e.Code + " " + e.Desc
41 | }
42 |
43 | // Parent returns the parent error
44 | func (e *FatalClientErr) Parent() error {
45 | return e.ParentErr
46 | }
47 |
48 | // NewClientErr creates a ClientErr with the supplied human and machine readable strings
49 | func NewFatalClientErr(parent error, code string, description string) *FatalClientErr {
50 | return &FatalClientErr{parent, code, description}
51 | }
52 |
--------------------------------------------------------------------------------
/coverage.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Generate test coverage statistics for Go packages.
3 | #
4 | # Works around the fact that `go test -coverprofile` currently does not work
5 | # with multiple packages, see https://code.google.com/p/go/issues/detail?id=6909
6 | #
7 | # Usage: coverage.sh [--html|--coveralls]
8 | #
9 | # --html Additionally create HTML report
10 | # --coveralls Push coverage statistics to coveralls.io
11 | #
12 |
13 | set -e
14 |
15 | workdir=.cover
16 | profile="$workdir/cover.out"
17 | mode=count
18 |
19 | generate_cover_data() {
20 | rm -rf "$workdir"
21 | mkdir "$workdir"
22 |
23 | for pkg in "$@"; do
24 | f="$workdir/$(echo $pkg | tr / -).cover"
25 | go test -covermode="$mode" -coverprofile="$f" "$pkg"
26 | done
27 |
28 | echo "mode: $mode" >"$profile"
29 | grep -h -v "^mode:" "$workdir"/*.cover >>"$profile"
30 | }
31 |
32 | show_html_report() {
33 | go tool cover -html="$profile" -o="$workdir"/coverage.html
34 | }
35 |
36 | show_csv_report() {
37 | go tool cover -func="$profile" -o="$workdir"/coverage.csv
38 | }
39 |
40 | push_to_coveralls() {
41 | echo "Pushing coverage statistics to coveralls.io"
42 | $HOME/gopath/bin/goveralls -coverprofile="$profile" -service=travis-ci -ignore="nsqadmin/bindata.go"
43 | }
44 |
45 | generate_cover_data $(go list ./...)
46 | show_csv_report
47 |
48 | case "$1" in
49 | "")
50 | ;;
51 | --html)
52 | show_html_report ;;
53 | --coveralls)
54 | push_to_coveralls ;;
55 | *)
56 | echo >&2 "error: invalid option: $1"; exit 1 ;;
57 | esac
58 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | PREFIX=/usr/local
2 | DESTDIR=
3 | GOFLAGS=
4 | BINDIR=${PREFIX}/bin
5 |
6 | BLDDIR = build
7 | EXT=
8 | ifeq (${GOOS},windows)
9 | EXT=.exe
10 | endif
11 |
12 | APPS = nsqd nsqlookupd nsqadmin nsq_pubsub nsq_to_nsq nsq_to_file nsq_to_http nsq_tail nsq_stat to_nsq
13 | all: $(APPS)
14 |
15 | $(BLDDIR)/nsqd: $(wildcard apps/nsqd/*.go nsqd/*.go nsq/*.go internal/*/*.go)
16 | $(BLDDIR)/nsqlookupd: $(wildcard apps/nsqlookupd/*.go nsqlookupd/*.go nsq/*.go internal/*/*.go)
17 | $(BLDDIR)/nsqadmin: $(wildcard apps/nsqadmin/*.go nsqadmin/*.go nsqadmin/templates/*.go internal/*/*.go)
18 | $(BLDDIR)/nsq_pubsub: $(wildcard apps/nsq_pubsub/*.go nsq/*.go internal/*/*.go)
19 | $(BLDDIR)/nsq_to_nsq: $(wildcard apps/nsq_to_nsq/*.go nsq/*.go internal/*/*.go)
20 | $(BLDDIR)/nsq_to_file: $(wildcard apps/nsq_to_file/*.go nsq/*.go internal/*/*.go)
21 | $(BLDDIR)/nsq_to_http: $(wildcard apps/nsq_to_http/*.go nsq/*.go internal/*/*.go)
22 | $(BLDDIR)/nsq_tail: $(wildcard apps/nsq_tail/*.go nsq/*.go internal/*/*.go)
23 | $(BLDDIR)/nsq_stat: $(wildcard apps/nsq_stat/*.go internal/*/*.go)
24 | $(BLDDIR)/to_nsq: $(wildcard apps/to_nsq/*.go internal/*/*.go)
25 |
26 | $(BLDDIR)/%:
27 | @mkdir -p $(dir $@)
28 | go build ${GOFLAGS} -o $@ ./apps/$*
29 |
30 | $(APPS): %: $(BLDDIR)/%
31 |
32 | clean:
33 | rm -fr $(BLDDIR)
34 |
35 | .PHONY: install clean all
36 | .PHONY: $(APPS)
37 |
38 | install: $(APPS)
39 | install -m 755 -d ${DESTDIR}${BINDIR}
40 | for APP in $^ ; do install -m 755 ${BLDDIR}/$$APP ${DESTDIR}${BINDIR}/$$APP${EXT} ; done
41 |
--------------------------------------------------------------------------------
/nsqd/test/cert.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # call this script with an email address (valid or not).
3 | # like:
4 | # ./cert.sh foo@foo.com
5 | # Found: https://gist.github.com/ncw/9253562#file-makecert-sh
6 |
7 | if [ "$1" == "" ]; then
8 | echo "Need email as argument"
9 | exit 1
10 | fi
11 |
12 | EMAIL=$1
13 |
14 | rm -rf tmp
15 | mkdir tmp
16 | cd tmp
17 |
18 | echo "make CA"
19 | PRIVKEY="test"
20 | openssl req -new -x509 -days 365 -keyout ca.key -out ca.pem -subj "/C=DE/ST=NRW/L=Earth/O=Random Company/OU=IT/CN=www.random.com/emailAddress=KryptoKings@random.com" -passout pass:$PRIVKEY
21 |
22 | echo "make server cert"
23 | openssl req -new -nodes -x509 -out server.pem -keyout server.key -days 3650 -subj "/C=DE/ST=NRW/L=Earth/O=Random Company/OU=IT/CN=www.random.com/emailAddress=${EMAIL}"
24 |
25 | echo "make client cert"
26 | #openssl req -new -nodes -x509 -out client.pem -keyout client.key
27 | #-days 3650 -subj "/C=DE/ST=NRW/L=Earth/O=Random
28 | #Company/OU=IT/CN=www.random.com/emailAddress=${EMAIL}"
29 |
30 | openssl genrsa -out client.key 2048
31 | echo "00" > ca.srl
32 | openssl req -sha1 -key client.key -new -out client.req -subj "/C=DE/ST=NRW/L=Earth/O=Random Company/OU=IT/CN=client.com/emailAddress=${EMAIL}"
33 | # Adding -addtrust clientAuth makes certificates Go can't read
34 | openssl x509 -req -days 365 -in client.req -CA ca.pem -CAkey ca.key -passin pass:$PRIVKEY -out client.pem # -addtrust clientAuth
35 |
36 | openssl x509 -extfile ../openssl.conf -extensions ssl_client -req -days 365 -in client.req -CA ca.pem -CAkey ca.key -passin pass:$PRIVKEY -out client.pem
37 |
38 | cd ..
39 | mv tmp/* certs
40 | rm -rf tmp
41 |
--------------------------------------------------------------------------------
/internal/pqueue/pqueue.go:
--------------------------------------------------------------------------------
1 | package pqueue
2 |
3 | import (
4 | "container/heap"
5 | )
6 |
7 | type Item struct {
8 | Value interface{}
9 | Priority int64
10 | Index int
11 | }
12 |
13 | // this is a priority queue as implemented by a min heap
14 | // ie. the 0th element is the *lowest* value
15 | type PriorityQueue []*Item
16 |
17 | func New(capacity int) PriorityQueue {
18 | return make(PriorityQueue, 0, capacity)
19 | }
20 |
21 | func (pq PriorityQueue) Len() int {
22 | return len(pq)
23 | }
24 |
25 | func (pq PriorityQueue) Less(i, j int) bool {
26 | return pq[i].Priority < pq[j].Priority
27 | }
28 |
29 | func (pq PriorityQueue) Swap(i, j int) {
30 | pq[i], pq[j] = pq[j], pq[i]
31 | pq[i].Index = i
32 | pq[j].Index = j
33 | }
34 |
35 | func (pq *PriorityQueue) Push(x interface{}) {
36 | n := len(*pq)
37 | c := cap(*pq)
38 | if n+1 > c {
39 | npq := make(PriorityQueue, n, c*2)
40 | copy(npq, *pq)
41 | *pq = npq
42 | }
43 | *pq = (*pq)[0 : n+1]
44 | item := x.(*Item)
45 | item.Index = n
46 | (*pq)[n] = item
47 | }
48 |
49 | func (pq *PriorityQueue) Pop() interface{} {
50 | n := len(*pq)
51 | c := cap(*pq)
52 | if n < (c/2) && c > 25 {
53 | npq := make(PriorityQueue, n, c/2)
54 | copy(npq, *pq)
55 | *pq = npq
56 | }
57 | item := (*pq)[n-1]
58 | item.Index = -1
59 | *pq = (*pq)[0 : n-1]
60 | return item
61 | }
62 |
63 | func (pq *PriorityQueue) PeekAndShift(max int64) (*Item, int64) {
64 | if pq.Len() == 0 {
65 | return nil, 0
66 | }
67 |
68 | item := (*pq)[0]
69 | if item.Priority > max {
70 | return nil, item.Priority - max
71 | }
72 | heap.Remove(pq, 0)
73 |
74 | return item, 0
75 | }
76 |
--------------------------------------------------------------------------------
/internal/test/assertions.go:
--------------------------------------------------------------------------------
1 | package test
2 |
3 | import (
4 | "path/filepath"
5 | "reflect"
6 | "runtime"
7 | "testing"
8 | )
9 |
10 | func Equal(t *testing.T, expected, actual interface{}) {
11 | if !reflect.DeepEqual(expected, actual) {
12 | _, file, line, _ := runtime.Caller(1)
13 | t.Logf("\033[31m%s:%d:\n\n\t %#v (expected)\n\n\t!= %#v (actual)\033[39m\n\n",
14 | filepath.Base(file), line, expected, actual)
15 | t.FailNow()
16 | }
17 | }
18 |
19 | func NotEqual(t *testing.T, expected, actual interface{}) {
20 | if reflect.DeepEqual(expected, actual) {
21 | _, file, line, _ := runtime.Caller(1)
22 | t.Logf("\033[31m%s:%d:\n\n\tnexp: %#v\n\n\tgot: %#v\033[39m\n\n",
23 | filepath.Base(file), line, expected, actual)
24 | t.FailNow()
25 | }
26 | }
27 |
28 | func Nil(t *testing.T, object interface{}) {
29 | if !isNil(object) {
30 | _, file, line, _ := runtime.Caller(1)
31 | t.Logf("\033[31m%s:%d:\n\n\t
12 | {{#if collection.length}}
13 |
14 |
29 | {{else}}
30 |
31 | {{/if}}
32 |
33 | | Topic | 16 | {{#if graph_active}}Depth | {{/if}} 17 | {{#if graph_active}}Messages | {{/if}} 18 | {{#if graph_active}}Rate | {{/if}} 19 |
|---|---|---|---|
| {{name}} | 23 | {{#if ../graph_active}}{{/if}} 26 | |
Notice
No Topics Found
5 |
9 |
10 |
6 |
8 | NSQd Nodes ({{collection.length}})
7 |
11 |
52 |
--------------------------------------------------------------------------------
/nsqd/in_flight_pqueue.go:
--------------------------------------------------------------------------------
1 | package nsqd
2 |
3 | type inFlightPqueue []*Message
4 |
5 | func newInFlightPqueue(capacity int) inFlightPqueue {
6 | return make(inFlightPqueue, 0, capacity)
7 | }
8 |
9 | func (pq inFlightPqueue) Swap(i, j int) {
10 | pq[i], pq[j] = pq[j], pq[i]
11 | pq[i].index = i
12 | pq[j].index = j
13 | }
14 |
15 | func (pq *inFlightPqueue) Push(x *Message) {
16 | n := len(*pq)
17 | c := cap(*pq)
18 | if n+1 > c {
19 | npq := make(inFlightPqueue, n, c*2)
20 | copy(npq, *pq)
21 | *pq = npq
22 | }
23 | *pq = (*pq)[0 : n+1]
24 | x.index = n
25 | (*pq)[n] = x
26 | pq.up(n)
27 | }
28 |
29 | func (pq *inFlightPqueue) Pop() *Message {
30 | n := len(*pq)
31 | c := cap(*pq)
32 | pq.Swap(0, n-1)
33 | pq.down(0, n-1)
34 | if n < (c/2) && c > 25 {
35 | npq := make(inFlightPqueue, n, c/2)
36 | copy(npq, *pq)
37 | *pq = npq
38 | }
39 | x := (*pq)[n-1]
40 | x.index = -1
41 | *pq = (*pq)[0 : n-1]
42 | return x
43 | }
44 |
45 | func (pq *inFlightPqueue) Remove(i int) *Message {
46 | n := len(*pq)
47 | if n-1 != i {
48 | pq.Swap(i, n-1)
49 | pq.down(i, n-1)
50 | pq.up(i)
51 | }
52 | x := (*pq)[n-1]
53 | x.index = -1
54 | *pq = (*pq)[0 : n-1]
55 | return x
56 | }
57 |
58 | func (pq *inFlightPqueue) PeekAndShift(max int64) (*Message, int64) {
59 | if len(*pq) == 0 {
60 | return nil, 0
61 | }
62 |
63 | x := (*pq)[0]
64 | if x.pri > max {
65 | return nil, x.pri - max
66 | }
67 | pq.Pop()
68 |
69 | return x, 0
70 | }
71 |
72 | func (pq *inFlightPqueue) up(j int) {
73 | for {
74 | i := (j - 1) / 2 // parent
75 | if i == j || (*pq)[j].pri >= (*pq)[i].pri {
76 | break
77 | }
78 | pq.Swap(i, j)
79 | j = i
80 | }
81 | }
82 |
83 | func (pq *inFlightPqueue) down(i, n int) {
84 | for {
85 | j1 := 2*i + 1
86 | if j1 >= n || j1 < 0 { // j1 < 0 after int overflow
87 | break
88 | }
89 | j := j1 // left child
90 | if j2 := j1 + 1; j2 < n && (*pq)[j1].pri >= (*pq)[j2].pri {
91 | j = j2 // = 2*i + 2 // right child
92 | }
93 | if (*pq)[j].pri >= (*pq)[i].pri {
94 | break
95 | }
96 | pq.Swap(i, j)
97 | i = j
98 | }
99 | }
100 |
--------------------------------------------------------------------------------
/nsqadmin/.eslintrc:
--------------------------------------------------------------------------------
1 | {
2 | "env": {
3 | "browser": true
4 | },
5 | // Rule docs: http://eslint.org/docs/rules/
6 | "rules": {
7 | "block-scoped-var": [1],
8 | "brace-style": [1, "1tbs", {"allowSingleLine": true}],
9 | "camelcase": [1],
10 | "comma-spacing": [1],
11 | "comma-style": [1],
12 | "computed-property-spacing": [1, "never"],
13 | "consistent-return": [1],
14 | "consistent-this": [1, "self"],
15 | "curly": [2],
16 | "dot-notation": [0],
17 | "eol-last": [1],
18 | "eqeqeq": [1],
19 | "indent": [1, 4],
20 | "key-spacing": [1],
21 | "max-len": [1, 100],
22 | "max-nested-callbacks": [2, 3], // ????
23 | "new-cap": [1],
24 | "new-parens": [1],
25 | "no-caller": [2],
26 | "no-console": [0],
27 | "no-eval": [2],
28 | "no-extend-native": [2],
29 | "no-extra-bind": [1],
30 | "no-floating-decimal": [1],
31 | "no-iterator": [1],
32 | "no-lone-blocks": [1],
33 | "no-lonely-if": [1],
34 | "no-mixed-requires": [0],
35 | "no-mixed-spaces-and-tabs": [1],
36 | "no-multi-spaces": [1],
37 | "no-multi-str": [1],
38 | "no-multiple-empty-lines": [2, {"max": 2}],
39 | "no-native-reassign": [1],
40 | "no-new": [0],
41 | "no-redeclare": [1],
42 | "no-shadow": [1],
43 | "no-spaced-func": [1],
44 | "no-throw-literal": [1],
45 | "no-trailing-spaces": [1],
46 | "no-undef": [1],
47 | "no-underscore-dangle": [0],
48 | "no-unneeded-ternary": [1],
49 | "no-unused-vars": [1],
50 | "no-use-before-define": [1, "nofunc"],
51 | "no-with": [2],
52 | "one-var": [1, "never"],
53 | "quotes": [1, "single"],
54 | "radix": [1],
55 | "semi": [1, "always"],
56 | "semi-spacing": [1],
57 | "space-after-function-name": [1, "never"],
58 | "space-after-keywords": [1, "always"],
59 | "space-before-blocks": [1, "always"],
60 | "space-before-function-paren": [1, "never"],
61 | "space-in-parens": [1, "never"],
62 | "space-infix-ops": [1],
63 | "space-return-throw-case": [1],
64 | "space-unary-ops": [1],
65 | "strict": [0],
66 | "wrap-iife": [1]
67 | },
68 | "globals": {
69 | "module": true,
70 | "require": true
71 | }
72 | }
73 |
--------------------------------------------------------------------------------
/bench/bench_channels/bench_channels.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bufio"
5 | "flag"
6 | "fmt"
7 | "net"
8 | "sync"
9 | "time"
10 |
11 | "github.com/nsqio/go-nsq"
12 | )
13 |
14 | var (
15 | num = flag.Int("num", 10000, "num channels")
16 | tcpAddress = flag.String("nsqd-tcp-address", "127.0.0.1:4150", "
12 |
13 |
50 |
51 | | Hostname | 15 |Broadcast Address | 16 |TCP Port | 17 |HTTP Port | 18 |Version | 19 | {{#if nsqlookupd.length}} 20 |Lookupd Conns. | 21 | {{/if}} 22 |Topics | 23 |
|---|---|---|---|---|---|---|
| {{hostname}} | 27 |{{broadcast_address}} | 28 |{{tcp_port}} | 29 |{{http_port}} | 30 |{{version}} | 31 | {{#if ../nsqlookupd.length}} 32 |33 | {{remote_addresses.length}} 34 | 37 | | 38 | {{/if}} 39 |40 | {{#if topics.length}} 41 | {{topics.length}} 42 | {{#each topics}} 43 | {{topic}} 44 | {{/each}} 45 | {{/if}} 46 | | 47 |
5 |
9 |
10 | {{#unless nsqlookupd.length}}
11 |
6 |
8 | Lookup
7 |
12 |
14 | {{else}}
15 | Notice
nsqadmin is not configured with nsqlookupd hosts 13 |
16 |
27 |
28 |
17 |
18 |
25 |
26 | | nsqlookupd Host | 20 |
|---|
| {{this}} |
29 |
53 |
54 |
30 | {{#if topics}}
31 |
50 | {{/if}}
51 |
52 |
32 | Below is a tree of Topics/Channels that are currently inactive (i.e. not produced on any nsqd in the cluster but are present in the lookup data)
33 |
34 | -
35 | {{#each topics}}
36 |
-
37 | {{name}}
38 |
-
39 | {{#each channels}}
40 |
- 41 | {{this}} 42 | 43 | {{/each}} 44 |
46 | {{/each}}
47 |
Notice
No inactive Topics
55 |
71 | {{/unless}}
72 |
--------------------------------------------------------------------------------
/nsqd/stats_test.go:
--------------------------------------------------------------------------------
1 | package nsqd
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "os"
7 | "strconv"
8 | "testing"
9 | "time"
10 |
11 | "github.com/mreiferson/go-snappystream"
12 | "github.com/nsqio/nsq/internal/http_api"
13 | "github.com/nsqio/nsq/internal/test"
14 | )
15 |
16 | func TestStats(t *testing.T) {
17 | opts := NewOptions()
18 | opts.Logger = test.NewTestLogger(t)
19 | tcpAddr, _, nsqd := mustStartNSQD(opts)
20 | defer os.RemoveAll(opts.DataPath)
21 | defer nsqd.Exit()
22 |
23 | topicName := "test_stats" + strconv.Itoa(int(time.Now().Unix()))
24 | topic := nsqd.GetTopic(topicName)
25 | msg := NewMessage(<-nsqd.idChan, []byte("test body"))
26 | topic.PutMessage(msg)
27 |
28 | conn, err := mustConnectNSQD(tcpAddr)
29 | test.Nil(t, err)
30 | defer conn.Close()
31 |
32 | identify(t, conn, nil, frameTypeResponse)
33 | sub(t, conn, topicName, "ch")
34 |
35 | stats := nsqd.GetStats()
36 | t.Logf("stats: %+v", stats)
37 |
38 | test.Equal(t, 1, len(stats))
39 | test.Equal(t, 1, len(stats[0].Channels))
40 | test.Equal(t, 1, len(stats[0].Channels[0].Clients))
41 | }
42 |
43 | func TestClientAttributes(t *testing.T) {
44 | userAgent := "Test User Agent"
45 |
46 | opts := NewOptions()
47 | opts.Logger = test.NewTestLogger(t)
48 | opts.Verbose = true
49 | opts.SnappyEnabled = true
50 | tcpAddr, httpAddr, nsqd := mustStartNSQD(opts)
51 | defer os.RemoveAll(opts.DataPath)
52 | defer nsqd.Exit()
53 |
54 | conn, err := mustConnectNSQD(tcpAddr)
55 | test.Nil(t, err)
56 | defer conn.Close()
57 |
58 | data := identify(t, conn, map[string]interface{}{
59 | "snappy": true,
60 | "user_agent": userAgent,
61 | }, frameTypeResponse)
62 | resp := struct {
63 | Snappy bool `json:"snappy"`
64 | UserAgent string `json:"user_agent"`
65 | }{}
66 | err = json.Unmarshal(data, &resp)
67 | test.Nil(t, err)
68 | test.Equal(t, true, resp.Snappy)
69 |
70 | r := snappystream.NewReader(conn, snappystream.SkipVerifyChecksum)
71 | w := snappystream.NewWriter(conn)
72 | readValidate(t, r, frameTypeResponse, "OK")
73 |
74 | topicName := "test_client_attributes" + strconv.Itoa(int(time.Now().Unix()))
75 | sub(t, readWriter{r, w}, topicName, "ch")
76 |
77 | var d struct {
78 | Topics []struct {
79 | Channels []struct {
80 | Clients []struct {
81 | UserAgent string `json:"user_agent"`
82 | Snappy bool `json:"snappy"`
83 | } `json:"clients"`
84 | } `json:"channels"`
85 | } `json:"topics"`
86 | }
87 |
88 | endpoint := fmt.Sprintf("http://127.0.0.1:%d/stats?format=json", httpAddr.Port)
89 | err = http_api.NewClient(nil, ConnectTimeout, RequestTimeout).GETV1(endpoint, &d)
90 | test.Nil(t, err)
91 |
92 | test.Equal(t, userAgent, d.Topics[0].Channels[0].Clients[0].UserAgent)
93 | test.Equal(t, true, d.Topics[0].Channels[0].Clients[0].Snappy)
94 | }
95 |
--------------------------------------------------------------------------------
/nsqd/message.go:
--------------------------------------------------------------------------------
1 | package nsqd
2 |
3 | import (
4 | "bytes"
5 | "encoding/binary"
6 | "fmt"
7 | "io"
8 | "time"
9 | )
10 |
11 | const (
12 | MsgIDLength = 16
13 | minValidMsgLength = MsgIDLength + 8 + 2 // Timestamp + Attempts
14 | )
15 |
16 | type MessageID [MsgIDLength]byte
17 |
18 | type Message struct {
19 | ID MessageID
20 | Body []byte
21 | Timestamp int64
22 | Attempts uint16
23 |
24 | // for in-flight handling
25 | deliveryTS time.Time
26 | clientID int64
27 | pri int64
28 | index int
29 | deferred time.Duration
30 | }
31 |
32 | func NewMessage(id MessageID, body []byte) *Message {
33 | return &Message{
34 | ID: id,
35 | Body: body,
36 | Timestamp: time.Now().UnixNano(),
37 | }
38 | }
39 |
40 | func (m *Message) WriteTo(w io.Writer) (int64, error) {
41 | var buf [10]byte
42 | var total int64
43 |
44 | binary.BigEndian.PutUint64(buf[:8], uint64(m.Timestamp))
45 | binary.BigEndian.PutUint16(buf[8:10], uint16(m.Attempts))
46 |
47 | n, err := w.Write(buf[:])
48 | total += int64(n)
49 | if err != nil {
50 | return total, err
51 | }
52 |
53 | n, err = w.Write(m.ID[:])
54 | total += int64(n)
55 | if err != nil {
56 | return total, err
57 | }
58 |
59 | n, err = w.Write(m.Body)
60 | total += int64(n)
61 | if err != nil {
62 | return total, err
63 | }
64 |
65 | return total, nil
66 | }
67 |
68 | // decodeMessage deserializes data (as []byte) and creates a new Message
69 | // message format:
70 | // [x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x]...
71 | // | (int64) || || (hex string encoded in ASCII) || (binary)
72 | // | 8-byte || || 16-byte || N-byte
73 | // ------------------------------------------------------------------------------------------...
74 | // nanosecond timestamp ^^ message ID message body
75 | // (uint16)
76 | // 2-byte
77 | // attempts
78 | func decodeMessage(b []byte) (*Message, error) {
79 | var msg Message
80 |
81 | if len(b) < minValidMsgLength {
82 | return nil, fmt.Errorf("invalid message buffer size (%d)", len(b))
83 | }
84 |
85 | msg.Timestamp = int64(binary.BigEndian.Uint64(b[:8]))
86 | msg.Attempts = binary.BigEndian.Uint16(b[8:10])
87 | copy(msg.ID[:], b[10:10+MsgIDLength])
88 | msg.Body = b[10+MsgIDLength:]
89 |
90 | return &msg, nil
91 | }
92 |
93 | func writeMessageToBackend(buf *bytes.Buffer, msg *Message, bq BackendQueue) error {
94 | buf.Reset()
95 | _, err := msg.WriteTo(buf)
96 | if err != nil {
97 | return err
98 | }
99 | return bq.Put(buf.Bytes())
100 | }
101 |
--------------------------------------------------------------------------------
/nsqadmin/static/js/views/lookup.js:
--------------------------------------------------------------------------------
1 | var _ = require('underscore');
2 | var $ = require('jquery');
3 |
4 | var AppState = require('../app_state');
5 | var Pubsub = require('../lib/pubsub');
6 | var BaseView = require('./base');
7 |
8 | var Topic = require('../models/topic');
9 | var Channel = require('../models/channel');
10 |
11 | var LookupView = BaseView.extend({
12 | className: 'lookup container-fluid',
13 |
14 | template: require('./spinner.hbs'),
15 |
16 | events: {
17 | 'click .hierarchy button': 'onCreateTopicChannel',
18 | 'click .delete-topic-link': 'onDeleteTopic',
19 | 'click .delete-channel-link': 'onDeleteChannel'
20 | },
21 |
22 | initialize: function() {
23 | BaseView.prototype.initialize.apply(this, arguments);
24 | $.ajax(AppState.url('/topics?inactive=true'))
25 | .done(function(data) {
26 | this.template = require('./lookup.hbs');
27 | this.render({
28 | 'topics': _.map(data['topics'], function(v, k) {
29 | return {'name': k, 'channels': v};
30 | }),
31 | 'message': data['message']
32 | });
33 | }.bind(this))
34 | .fail(this.handleViewError.bind(this))
35 | .always(Pubsub.trigger.bind(Pubsub, 'view:ready'));
36 | },
37 |
38 | onCreateTopicChannel: function(e) {
39 | e.preventDefault();
40 | e.stopPropagation();
41 | var topic = $(e.target.form.elements['topic']).val();
42 | var channel = $(e.target.form.elements['channel']).val();
43 | if (topic === '' && channel === '') {
44 | return;
45 | }
46 | $.post(AppState.url('/topics'), JSON.stringify({
47 | 'topic': topic,
48 | 'channel': channel
49 | }))
50 | .done(function() { window.location.reload(true); })
51 | .fail(this.handleAJAXError.bind(this));
52 | },
53 |
54 | onDeleteTopic: function(e) {
55 | e.preventDefault();
56 | e.stopPropagation();
57 | var topic = new Topic({
58 | 'name': $(e.target).data('topic')
59 | });
60 | topic.destroy({'dataType': 'text'})
61 | .done(function() { window.location.reload(true); })
62 | .fail(this.handleAJAXError.bind(this));
63 | },
64 |
65 | onDeleteChannel: function(e) {
66 | e.preventDefault();
67 | e.stopPropagation();
68 | var channel = new Channel({
69 | 'topic': $(e.target).data('topic'),
70 | 'name': $(e.target).data('channel')
71 | });
72 | channel.destroy({'dataType': 'text'})
73 | .done(function() { window.location.reload(true); })
74 | .fail(this.handleAJAXError.bind(this));
75 | }
76 | });
77 |
78 | module.exports = LookupView;
79 |
--------------------------------------------------------------------------------
/bench/bench_writer/bench_writer.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bufio"
5 | "flag"
6 | "log"
7 | "net"
8 | "runtime"
9 | "sync"
10 | "sync/atomic"
11 | "time"
12 |
13 | "github.com/nsqio/go-nsq"
14 | )
15 |
16 | var (
17 | runfor = flag.Duration("runfor", 10*time.Second, "duration of time to run")
18 | tcpAddress = flag.String("nsqd-tcp-address", "127.0.0.1:4150", "
56 |
69 |
70 |
201 | ![]()
210 | {{/if}}
211 |
212 |
--------------------------------------------------------------------------------
/nsqadmin/gulpfile.js:
--------------------------------------------------------------------------------
1 | var browserify = require('browserify');
2 | var clean = require('gulp-clean');
3 | var gulp = require('gulp');
4 | var notify = require('gulp-notify');
5 | var path = require('path');
6 | var sass = require('gulp-sass');
7 | var source = require('vinyl-source-stream');
8 | var taskListing = require('gulp-task-listing');
9 | var uglify = require('gulp-uglify');
10 | var sourcemaps = require('gulp-sourcemaps');
11 | var buffer = require('vinyl-buffer');
12 |
13 |
14 | var ROOT = 'static';
15 |
16 | var VENDOR_CONFIG = {
17 | 'src': [
18 | 'backbone',
19 | 'jquery',
20 | 'underscore',
21 | 'bootbox',
22 | ],
23 | 'target': 'vendor.js',
24 | 'targetDir': './static/build/'
25 | };
26 |
27 | function excludeVendor(b) {
28 | VENDOR_CONFIG.src.forEach(function(vendorLib) {
29 | b.exclude(vendorLib);
30 | });
31 | }
32 |
33 | function bytesToKB(bytes) { return Math.floor(+bytes/1024); }
34 |
35 | function logBundle(filename, watching) {
36 | return function (err, buf) {
37 | if (err) {
38 | console.error(err.toString());
39 | if (!watching) {
40 | process.exit(1);
41 | }
42 | }
43 | if (!watching) {
44 | console.log(filename + ' ' + bytesToKB(buf.length) + ' KB written');
45 | }
46 | }
47 | }
48 |
49 |
50 | function sassTask(root, inputFile) {
51 | return function() {
52 | var onError = function(err) {
53 | notify({'title': 'Sass Compile Error'}).write(err);
54 | };
55 | gulp.src(path.join(root, 'css', inputFile))
56 | .pipe(sass({
57 | 'sourceComments': 'map',
58 | 'onError': onError
59 | }))
60 | .pipe(gulp.dest(path.join(root, 'build/')));
61 | };
62 | }
63 |
64 |
65 | function browserifyTask(root, inputFile) {
66 | return function() {
67 | var onError = function() {
68 | var args = Array.prototype.slice.call(arguments);
69 | notify.onError({
70 | 'title': 'JS Compile Error',
71 | 'message': '<%= error.message %>'
72 | }).apply(this, args);
73 | // Keep gulp from hanging on this task
74 | this.emit('end');
75 | };
76 |
77 | // Browserify needs a node module to import as its arg, so we need to
78 | // force the leading "./" to be included.
79 | var b = browserify({
80 | entries: './' + path.join(root, 'js', inputFile),
81 | debug: true
82 | })
83 |
84 | excludeVendor(b);
85 |
86 | return b.bundle()
87 | .pipe(source(inputFile))
88 | .pipe(buffer())
89 | .pipe(sourcemaps.init({'loadMaps': true, 'debug': true}))
90 | // Add transformation tasks to the pipeline here.
91 | .pipe(uglify())
92 | .on('error', onError)
93 | .pipe(sourcemaps.write('./'))
94 | .pipe(gulp.dest(path.join(root, 'build/')));
95 | };
96 | }
97 |
98 |
99 | function watchTask(root) {
100 | return function() {
101 | gulp.watch(path.join(root, 'sass/**/*.scss'), ['sass']);
102 | gulp.watch([
103 | path.join(root, 'js/**/*.js'),
104 | path.join(root, 'js/**/*.hbs')
105 | ], ['browserify']);
106 | gulp.watch([
107 | path.join(root, 'html/**'),
108 | path.join(root, 'fonts/**')
109 | ], ['sync-static-assets'])
110 | };
111 | }
112 |
113 |
114 | function cleanTask(){
115 | var paths = Array.prototype.slice.apply(arguments);
116 | return function () {
117 | gulp.src(paths).pipe(clean());
118 | };
119 | }
120 |
121 | gulp.task('vendor-build-js', function() {
122 | var onError = function() {
123 | var args = Array.prototype.slice.call(arguments);
124 | notify.onError({
125 | 'title': 'JS Compile Error',
126 | 'message': '<%= error.message %>'
127 | }).apply(this, args);
128 | // Keep gulp from hanging on this task
129 | this.emit('end');
130 | };
131 |
132 | var b = browserify()
133 | .require(VENDOR_CONFIG.src);
134 |
135 | return b.bundle(logBundle(VENDOR_CONFIG.target))
136 | .pipe(source(VENDOR_CONFIG.target))
137 | .pipe(buffer())
138 | // Add transformation tasks to the pipeline here.
139 | .pipe(uglify())
140 | .on('error', onError)
141 | .pipe(gulp.dest(VENDOR_CONFIG.targetDir));
142 | });
143 |
144 | gulp.task('help', taskListing);
145 |
146 | gulp.task('sync-static-assets', function() {
147 | return gulp.src([
148 | path.join(ROOT, 'html/**'),
149 | path.join(ROOT, 'fonts/**'),
150 | path.join(ROOT, 'img/**')
151 | ]).pipe(gulp.dest(path.join(ROOT, 'build')));
152 | });
153 |
154 | gulp.task('sass', sassTask(ROOT, '*.*css'));
155 | gulp.task('browserify', browserifyTask(ROOT, 'main.js'));
156 | gulp.task('build', ['sass', 'browserify', 'sync-static-assets', 'vendor-build-js']);
157 | gulp.task('watch', ['build'], watchTask(ROOT));
158 | gulp.task('clean', cleanTask(path.join(ROOT, 'build')));
159 |
160 | gulp.task('default', ['help']);
161 |
--------------------------------------------------------------------------------
/nsqd/options.go:
--------------------------------------------------------------------------------
1 | package nsqd
2 |
3 | import (
4 | "crypto/md5"
5 | "crypto/tls"
6 | "hash/crc32"
7 | "io"
8 | "log"
9 | "os"
10 | "time"
11 | )
12 |
13 | type Options struct {
14 | // basic options
15 | ID int64 `flag:"worker-id" cfg:"id"`
16 | Verbose bool `flag:"verbose"`
17 | TCPAddress string `flag:"tcp-address"`
18 | HTTPAddress string `flag:"http-address"`
19 | HTTPSAddress string `flag:"https-address"`
20 | BroadcastAddress string `flag:"broadcast-address"`
21 | NSQLookupdTCPAddresses []string `flag:"lookupd-tcp-address" cfg:"nsqlookupd_tcp_addresses"`
22 | AuthHTTPAddresses []string `flag:"auth-http-address" cfg:"auth_http_addresses"`
23 | HTTPClientConnectTimeout time.Duration `flag:"http-client-connect-timeout" cfg:"http_client_connect_timeout"`
24 | HTTPClientRequestTimeout time.Duration `flag:"http-client-request-timeout" cfg:"http_client_request_timeout"`
25 |
26 | // diskqueue options
27 | DataPath string `flag:"data-path"`
28 | MemQueueSize int64 `flag:"mem-queue-size"`
29 | MaxBytesPerFile int64 `flag:"max-bytes-per-file"`
30 | SyncEvery int64 `flag:"sync-every"`
31 | SyncTimeout time.Duration `flag:"sync-timeout"`
32 |
33 | QueueScanInterval time.Duration
34 | QueueScanRefreshInterval time.Duration
35 | QueueScanSelectionCount int
36 | QueueScanWorkerPoolMax int
37 | QueueScanDirtyPercent float64
38 |
39 | // msg and command options
40 | MsgTimeout time.Duration `flag:"msg-timeout" arg:"1ms"`
41 | MaxMsgTimeout time.Duration `flag:"max-msg-timeout"`
42 | MaxMsgSize int64 `flag:"max-msg-size" deprecated:"max-message-size" cfg:"max_msg_size"`
43 | MaxBodySize int64 `flag:"max-body-size"`
44 | MaxReqTimeout time.Duration `flag:"max-req-timeout"`
45 | ClientTimeout time.Duration
46 |
47 | // client overridable configuration options
48 | MaxHeartbeatInterval time.Duration `flag:"max-heartbeat-interval"`
49 | MaxRdyCount int64 `flag:"max-rdy-count"`
50 | MaxOutputBufferSize int64 `flag:"max-output-buffer-size"`
51 | MaxOutputBufferTimeout time.Duration `flag:"max-output-buffer-timeout"`
52 |
53 | // statsd integration
54 | StatsdAddress string `flag:"statsd-address"`
55 | StatsdPrefix string `flag:"statsd-prefix"`
56 | StatsdInterval time.Duration `flag:"statsd-interval" arg:"1s"`
57 | StatsdMemStats bool `flag:"statsd-mem-stats"`
58 |
59 | // e2e message latency
60 | E2EProcessingLatencyWindowTime time.Duration `flag:"e2e-processing-latency-window-time"`
61 | E2EProcessingLatencyPercentiles []float64 `flag:"e2e-processing-latency-percentile" cfg:"e2e_processing_latency_percentiles"`
62 |
63 | // TLS config
64 | TLSCert string `flag:"tls-cert"`
65 | TLSKey string `flag:"tls-key"`
66 | TLSClientAuthPolicy string `flag:"tls-client-auth-policy"`
67 | TLSRootCAFile string `flag:"tls-root-ca-file"`
68 | TLSRequired int `flag:"tls-required"`
69 | TLSMinVersion uint16 `flag:"tls-min-version"`
70 |
71 | // compression
72 | DeflateEnabled bool `flag:"deflate"`
73 | MaxDeflateLevel int `flag:"max-deflate-level"`
74 | SnappyEnabled bool `flag:"snappy"`
75 |
76 | Logger Logger
77 | }
78 |
79 | func NewOptions() *Options {
80 | hostname, err := os.Hostname()
81 | if err != nil {
82 | log.Fatal(err)
83 | }
84 |
85 | h := md5.New()
86 | io.WriteString(h, hostname)
87 | defaultID := int64(crc32.ChecksumIEEE(h.Sum(nil)) % 1024)
88 |
89 | return &Options{
90 | ID: defaultID,
91 |
92 | TCPAddress: "0.0.0.0:4150",
93 | HTTPAddress: "0.0.0.0:4151",
94 | HTTPSAddress: "0.0.0.0:4152",
95 | BroadcastAddress: hostname,
96 |
97 | NSQLookupdTCPAddresses: make([]string, 0),
98 | AuthHTTPAddresses: make([]string, 0),
99 |
100 | HTTPClientConnectTimeout: 2 * time.Second,
101 | HTTPClientRequestTimeout: 5 * time.Second,
102 |
103 | MemQueueSize: 10000,
104 | MaxBytesPerFile: 100 * 1024 * 1024,
105 | SyncEvery: 2500,
106 | SyncTimeout: 2 * time.Second,
107 |
108 | QueueScanInterval: 100 * time.Millisecond,
109 | QueueScanRefreshInterval: 5 * time.Second,
110 | QueueScanSelectionCount: 20,
111 | QueueScanWorkerPoolMax: 4,
112 | QueueScanDirtyPercent: 0.25,
113 |
114 | MsgTimeout: 60 * time.Second,
115 | MaxMsgTimeout: 15 * time.Minute,
116 | MaxMsgSize: 1024 * 1024,
117 | MaxBodySize: 5 * 1024 * 1024,
118 | MaxReqTimeout: 1 * time.Hour,
119 | ClientTimeout: 60 * time.Second,
120 |
121 | MaxHeartbeatInterval: 60 * time.Second,
122 | MaxRdyCount: 2500,
123 | MaxOutputBufferSize: 64 * 1024,
124 | MaxOutputBufferTimeout: 1 * time.Second,
125 |
126 | StatsdPrefix: "nsq.%s",
127 | StatsdInterval: 60 * time.Second,
128 | StatsdMemStats: true,
129 |
130 | E2EProcessingLatencyWindowTime: time.Duration(10 * time.Minute),
131 |
132 | DeflateEnabled: true,
133 | MaxDeflateLevel: 6,
134 | SnappyEnabled: true,
135 |
136 | TLSMinVersion: tls.VersionTLS10,
137 |
138 | Logger: log.New(os.Stderr, "[nsqd] ", log.Ldate|log.Ltime|log.Lmicroseconds),
139 | }
140 | }
141 |
--------------------------------------------------------------------------------
202 |
203 | Messages Processed
204 | 205 | {{#if graph_active}} 206 | 209 |