to listen on for HTTP clients
5 | http_address = "0.0.0.0:4161"
6 |
7 | ## rpc port used for cluster communication
8 | rpc_port = "4260"
9 | ## address that will be registered with lookupd (defaults to the OS hostname)
10 | #broadcast_address = ""
11 | ## the network interface for broadcast, the ip will be detected automatically.
12 | # use this configure instead of broadcast_address to keep all the configure is the same
13 | broadcast_interface = "eth0"
14 |
15 | cluster_id = "test-jepsen-dev-1"
16 | ## the etcd cluster ip list
17 | cluster_leadership_addresses = "http://etcd0.example.com:2379,http://etcd1.example.com:2379"
18 | # cluster_leadership_username = "username"
19 | # cluster_leadership_password = "password"
20 | ## default value is "NSQMetaData"
21 | # cluster_leadership_root_dir = "etcd_root_dir"
22 |
23 | ## duration of time a producer will remain in the active list since its last ping
24 | inactive_producer_timeout = "100s"
25 | # should at least twice as the ping interval on nsqd
26 | nsqd_ping_timeout= "15s"
27 |
28 | ## duration of time a producer will remain tombstoned if registration remains
29 | tombstone_lifetime = "45s"
30 |
31 | ## the detail of the log, larger number means more details
32 | log_level = 3
33 |
34 | ## if empty, use the default flag value in glog
35 | log_dir = "/var/log"
36 |
37 | ## the time period (in hour) that the balance is allowed.
38 | balance_interval = ["4", "5"]
39 |
--------------------------------------------------------------------------------
/internal/flume_log/flume_sdk.go:
--------------------------------------------------------------------------------
1 | package flume_log
2 |
3 | import (
4 | "errors"
5 | "strings"
6 | )
7 |
8 | var defaultAgentAddr string
9 |
10 | type FlumeLogger struct {
11 | client *FlumeClient
12 | }
13 |
14 | func init() {
15 | defaultAgentAddr = "127.0.0.1:5140"
16 | }
17 |
18 | func NewFlumeLogger() *FlumeLogger {
19 | c := NewFlumeClient(defaultAgentAddr)
20 | return &FlumeLogger{c}
21 | }
22 |
23 | func NewFlumeLoggerWithAddr(agentAddr string) *FlumeLogger {
24 | c := NewFlumeClient(agentAddr)
25 | return &FlumeLogger{c}
26 | }
27 |
28 | func (l *FlumeLogger) Info(logInfo string, detail *DetailInfo) error {
29 | return l.log("info", logInfo, detail)
30 | }
31 |
32 | func (l *FlumeLogger) Warn(logInfo string, detail *DetailInfo) error {
33 | return l.log("warn", logInfo, detail)
34 | }
35 |
36 | func (l *FlumeLogger) Error(logInfo string, detail *DetailInfo) error {
37 | return l.log("error", logInfo, detail)
38 | }
39 |
40 | func (l *FlumeLogger) log(typeInfo string, logInfo string, detail *DetailInfo) error {
41 | log_info := NewLogInfo()
42 | log_info.module = detail.module
43 | topic := "log." + gAppName + "." + detail.module
44 | log_info.topic = strings.ToLower(topic)
45 | log_info.level = typeInfo
46 | log_info.typ = typeInfo
47 | log_info.tag = logInfo
48 | log_info.detail = detail
49 | if l.client != nil {
50 | return l.client.SendLog(log_info.Serialize())
51 | }
52 | return errors.New("no client")
53 | }
54 |
55 | func (l *FlumeLogger) Stop() {
56 | if l.client != nil {
57 | l.client.Stop()
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/nsqlookupd_migrate/README.md:
--------------------------------------------------------------------------------
1 | #nsqlookupd migrate proxy
2 | Migrate proxy to help migrate from old nsq to youzan nsq.
3 |
4 | Migrate works as proxy of nsqlookupd of nsq and youzan nsq. It delegates lookup requests to nsqlookupd in old and new,
5 | migrate response and return to nsq client, according to migrate status read from memory.
6 |
7 | Migrate status defines as follow:
8 | const (
9 | M_OFF = iota //0. migrate not start
10 | M_CSR //1. return old&new lookup response for consumer, and old lookup response for producer
11 | M_CSR_PDR //2. return old&nsq lookup response for consumer, and new lookup response for producer
12 | M_FIN //3. return new lookup response for consumer&producer
13 | )
14 |
15 | > Note: migrate from youzan nsq to another does not support, since partition info in two youzan nsq cold not mix.
16 |
17 | > As to implementation of topic switches, current version shipped with topic switches in memory, switches could be updated via
18 | POST '$host:$port/switch' with body {"$topic":[0-3]}, and default switches for all topic is 0(M_OFF)
19 |
20 | ##Deploy
21 | start with config file:
22 | nsqlookupd_migrate -config=conf/nsqlookup_migrate.conf
23 |
24 | ##Config
25 | properties defined in nsqlookup migrate:
26 |
27 | origin-lookupd-http = "http://origin.nsqlookupd:4161"
28 | target-lookupd-http = "http://target.nsqlookupd:4161"
29 | env = "qa"
30 | log-level = 2
31 | log-dir = "/data/logs/nsqlookup_migrate"
32 | migrate-key = "origin.to.target"
33 |
--------------------------------------------------------------------------------
/consistence/etcd_utils.go:
--------------------------------------------------------------------------------
1 | package consistence
2 |
3 | import (
4 | "net/url"
5 | "os"
6 |
7 | "github.com/coreos/etcd/client"
8 | )
9 |
10 | const (
11 | ErrCodeEtcdNotReachable = 501
12 | ErrCodeUnhandledHTTPStatus = 502
13 | )
14 |
15 | var (
16 | hostname string
17 | ip string
18 | )
19 |
20 | func initEtcdPeers(machines []string) error {
21 | for i, ep := range machines {
22 | u, err := url.Parse(ep)
23 | if err != nil {
24 | return err
25 | }
26 | if u.Scheme == "" {
27 | u.Scheme = "http"
28 | }
29 | machines[i] = u.String()
30 | }
31 | return nil
32 | }
33 |
34 | func IsEtcdNotReachable(err error) bool {
35 | if cErr, ok := err.(client.Error); ok {
36 | return cErr.Code == ErrCodeEtcdNotReachable
37 | }
38 | return false
39 | }
40 |
41 | func IsEtcdWatchExpired(err error) bool {
42 | return isEtcdErrorNum(err, client.ErrorCodeEventIndexCleared)
43 | }
44 |
45 | func CheckKeyIfExist(err error) bool {
46 | return isEtcdErrorNum(err, client.ErrorCodeKeyNotFound)
47 | }
48 |
49 | func IsEtcdNotFile(err error) bool {
50 | return isEtcdErrorNum(err, client.ErrorCodeNotFile)
51 | }
52 |
53 | func IsEtcdNodeExist(err error) bool {
54 | return isEtcdErrorNum(err, client.ErrorCodeNodeExist)
55 | }
56 |
57 | func isEtcdErrorNum(err error, errorCode int) bool {
58 | if err != nil {
59 | if etcdError, ok := err.(client.Error); ok {
60 | return etcdError.Code == errorCode
61 | }
62 | // NOTE: There are other error types returned
63 | }
64 | return false
65 | }
66 |
67 | func init() {
68 | hostname, _ = os.Hostname()
69 | }
70 |
--------------------------------------------------------------------------------
/dist.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # 1. commit to bump the version and update the changelog/readme
4 | # 2. tag that commit
5 | # 3. use dist.sh to produce tar.gz for linux and darwin
6 | # 4. upload *.tar.gz to our bitly s3 bucket
7 | # 5. docker push nsqio/nsq
8 | # 6. push to nsqio/master
9 | # 7. update the release metadata on github / upload the binaries there too
10 | # 8. update the gh-pages branch with versions / download links
11 | # 9. update homebrew version
12 | # 10. send release announcement emails
13 | # 11. update IRC channel topic
14 | # 12. tweet
15 |
16 | set -e
17 |
18 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
19 | echo $GOPATH
20 |
21 | arch='amd64'
22 | version=$(awk '/const Binary/ {print $NF}' < $DIR/internal/version/binary.go | sed 's/"//g')
23 | goversion=$(go version | awk '{print $3}')
24 |
25 | for os in linux darwin windows; do
26 | echo "... building v$version for $os/$arch"
27 | BUILD=$(mktemp -d -t nsqXXXXXX)
28 | TARGET="nsq-$version.$os-$arch.$goversion"
29 | GOOS=$os GOARCH=$arch CGO_ENABLED=0 \
30 | make DESTDIR=$BUILD PREFIX=/$TARGET install
31 | pushd $BUILD
32 | if [ "$os" == "linux" ]; then
33 | cp -r $TARGET/bin $DIR/dist/docker/
34 | fi
35 | tar czvf $TARGET.tar.gz $TARGET
36 | mv $TARGET.tar.gz $DIR/dist
37 | popd
38 | make clean
39 | rm -r $BUILD
40 | done
41 |
42 | docker build -t nsqio/nsq:v$version .
43 | if [[ ! $version == *"-"* ]]; then
44 | echo "Tagging nsqio/nsq:v$version as the latest release."
45 | docker tag -f nsqio/nsq:v$version nsqio/nsq:latest
46 | fi
47 |
--------------------------------------------------------------------------------
/nsqlookupd_migrate/logger.go:
--------------------------------------------------------------------------------
1 | package nsqlookupd_migrate
2 |
3 | import (
4 | "fmt"
5 | "sync/atomic"
6 |
7 | "github.com/absolute8511/glog"
8 | )
9 |
10 | const (
11 | LOG_ERR int32 = iota
12 | LOG_WARN
13 | LOG_INFO
14 | LOG_DEBUG
15 | )
16 |
17 | type MigrateLogger struct {
18 | level int32
19 | }
20 |
21 | func (logger *MigrateLogger) SetLevel(level int32) {
22 | atomic.StoreInt32(&logger.level, level)
23 | glog.Infof("Log level set to %v", level)
24 | }
25 |
26 | func (logger *MigrateLogger) Level() int32 {
27 | return atomic.LoadInt32(&logger.level)
28 | }
29 |
30 | func (logger *MigrateLogger) AccessTrace(format string, v ...interface{}) {
31 | if logger.Level() >= LOG_INFO {
32 | accessFmt := fmt.Sprintf("[access] %v", format)
33 | glog.InfoDepth(2, fmt.Sprintf(accessFmt, v...))
34 | }
35 | }
36 |
37 | func (logger *MigrateLogger) Debug(format string, v ...interface{}) {
38 | if logger.Level() >= LOG_DEBUG {
39 | glog.InfoDepth(2, fmt.Sprintf(format, v...))
40 | }
41 | }
42 |
43 | func (logger *MigrateLogger) Info(format string, v ...interface{}) {
44 | if logger.Level() >= LOG_INFO {
45 | glog.InfoDepth(2, fmt.Sprintf(format, v...))
46 | }
47 | }
48 |
49 | func (logger *MigrateLogger) Warn(format string, v ...interface{}) {
50 | glog.WarningDepth(2, fmt.Sprintf(format, v...))
51 | }
52 |
53 | func (logger *MigrateLogger) Error(format string, v ...interface{}) {
54 | glog.ErrorDepth(2, fmt.Sprintf(format, v...))
55 | }
56 |
57 | func NewMigrateLogger(level int32) *MigrateLogger {
58 | mLog := MigrateLogger{
59 | level,
60 | }
61 | return &mLog
62 | }
63 |
--------------------------------------------------------------------------------
/nsqlookupd/lookup_protocol_v1_test.go:
--------------------------------------------------------------------------------
1 | package nsqlookupd
2 |
3 | import (
4 | "errors"
5 | "testing"
6 | "time"
7 |
8 | "github.com/youzan/nsq/internal/protocol"
9 | "github.com/youzan/nsq/internal/test"
10 | )
11 |
12 | func TestIOLoopReturnsClientErrWhenSendFails(t *testing.T) {
13 | fakeConn := test.NewFakeNetConn()
14 | fakeConn.WriteFunc = func(b []byte) (int, error) {
15 | return 0, errors.New("write error")
16 | }
17 |
18 | testIOLoopReturnsClientErr(t, fakeConn)
19 | }
20 |
21 | func TestIOLoopReturnsClientErrWhenSendSucceeds(t *testing.T) {
22 | fakeConn := test.NewFakeNetConn()
23 | fakeConn.WriteFunc = func(b []byte) (int, error) {
24 | return len(b), nil
25 | }
26 |
27 | testIOLoopReturnsClientErr(t, fakeConn)
28 | }
29 |
30 | func testIOLoopReturnsClientErr(t *testing.T, fakeConn test.FakeNetConn) {
31 | fakeConn.ReadFunc = func(b []byte) (int, error) {
32 | return copy(b, []byte("INVALID_COMMAND\n")), nil
33 | }
34 |
35 | opts := NewOptions()
36 | opts.Logger = newTestLogger(t)
37 |
38 | prot := &LookupProtocolV1{ctx: &Context{nsqlookupd: New(opts)}}
39 |
40 | errChan := make(chan error)
41 | test := func() {
42 | errChan <- prot.IOLoop(fakeConn)
43 | defer prot.ctx.nsqlookupd.Exit()
44 | }
45 | go test()
46 |
47 | var err error
48 | var timeout bool
49 |
50 | select {
51 | case err = <-errChan:
52 | case <-time.After(2 * time.Second):
53 | timeout = true
54 | }
55 |
56 | equal(t, timeout, false)
57 |
58 | nequal(t, err, nil)
59 | equal(t, err.Error(), "E_INVALID invalid command INVALID_COMMAND")
60 | nequal(t, err.(*protocol.FatalClientErr), nil)
61 | }
62 |
--------------------------------------------------------------------------------
/nsqadmin/static/js/views/nodes.js:
--------------------------------------------------------------------------------
1 | var $ = require('jquery');
2 |
3 | var Pubsub = require('../lib/pubsub');
4 | var AppState = require('../app_state');
5 |
6 | var BaseView = require('./base');
7 |
8 | var Nodes = require('../collections/nodes');
9 |
10 | var NodesView = BaseView.extend({
11 | className: 'nodes container-fluid',
12 |
13 | template: require('./spinner.hbs'),
14 |
15 | events: {
16 | 'click .conn-count': 'onClickConnCount'
17 | },
18 |
19 | initialize: function() {
20 | BaseView.prototype.initialize.apply(this, arguments);
21 | this.listenTo(AppState, 'change:graph_interval', this.render);
22 | this.collection = new Nodes();
23 | this.collection.fetch()
24 | .done(function(data) {
25 | this.template = require('./nodes.hbs');
26 | $.ajax({
27 | url: AppState.url('/cluster/stats'),
28 | success: function(data){
29 | clustersInfo = data;
30 | },
31 | async: false
32 | });
33 | this.render({
34 | 'message': data['message'],
35 | 'clusters': clustersInfo['clustersInfo']
36 | });
37 | }.bind(this))
38 | .fail(this.handleViewError.bind(this))
39 | .always(Pubsub.trigger.bind(Pubsub, 'view:ready'));
40 | },
41 |
42 | onClickConnCount: function(e) {
43 | e.preventDefault();
44 | $(e.target).next().toggle();
45 | }
46 | });
47 |
48 | module.exports = NodesView;
49 |
--------------------------------------------------------------------------------
/nsqdserver/test/cert.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # call this script with an email address (valid or not).
3 | # like:
4 | # ./cert.sh foo@foo.com
5 | # Found: https://gist.github.com/ncw/9253562#file-makecert-sh
6 |
7 | if [ "$1" == "" ]; then
8 | echo "Need email as argument"
9 | exit 1
10 | fi
11 |
12 | EMAIL=$1
13 |
14 | rm -rf tmp
15 | mkdir tmp
16 | cd tmp
17 |
18 | echo "make CA"
19 | PRIVKEY="test"
20 | openssl req -new -x509 -days 365 -keyout ca.key -out ca.pem -subj "/C=DE/ST=NRW/L=Earth/O=Random Company/OU=IT/CN=www.random.com/emailAddress=KryptoKings@random.com" -passout pass:$PRIVKEY
21 |
22 | echo "make server cert"
23 | openssl req -new -nodes -x509 -out server.pem -keyout server.key -days 3650 -subj "/C=DE/ST=NRW/L=Earth/O=Random Company/OU=IT/CN=www.random.com/emailAddress=${EMAIL}"
24 |
25 | echo "make client cert"
26 | #openssl req -new -nodes -x509 -out client.pem -keyout client.key
27 | #-days 3650 -subj "/C=DE/ST=NRW/L=Earth/O=Random
28 | #Company/OU=IT/CN=www.random.com/emailAddress=${EMAIL}"
29 |
30 | openssl genrsa -out client.key 2048
31 | echo "00" > ca.srl
32 | openssl req -sha1 -key client.key -new -out client.req -subj "/C=DE/ST=NRW/L=Earth/O=Random Company/OU=IT/CN=client.com/emailAddress=${EMAIL}"
33 | # Adding -addtrust clientAuth makes certificates Go can't read
34 | openssl x509 -req -days 365 -in client.req -CA ca.pem -CAkey ca.key -passin pass:$PRIVKEY -out client.pem # -addtrust clientAuth
35 |
36 | openssl x509 -extfile ../openssl.conf -extensions ssl_client -req -days 365 -in client.req -CA ca.pem -CAkey ca.key -passin pass:$PRIVKEY -out client.pem
37 |
38 | cd ..
39 | mv tmp/* certs
40 | rm -rf tmp
41 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing
2 |
3 | Thanks for your interest in contributing to NSQ!
4 |
5 | ## Code of Conduct
6 |
7 | Help us keep NSQ open and inclusive. Please read and follow our [Code of Conduct](CODE_OF_CONDUCT.md).
8 |
9 | ## Getting Started
10 |
11 | * make sure you have a [GitHub account](https://github.com/signup/free)
12 | * submit a ticket for your issue, assuming one does not already exist
13 | * clearly describe the issue including steps to reproduce when it is a bug
14 | * identify specific versions of the binaries and client libraries
15 | * fork the repository on GitHub
16 |
17 | ## Making Changes
18 |
19 | * create a branch from where you want to base your work
20 | * we typically name branches according to the following format: `helpful_name_`
21 | * make commits of logical units
22 | * make sure your commit messages are in a clear and readable format, example:
23 |
24 | ```
25 | nsqd: fixed bug in protocol_v2
26 |
27 | * update the message pump to properly account for RDYness
28 | * cleanup variable names
29 | * ...
30 | ```
31 |
32 | * if you're fixing a bug or adding functionality it probably makes sense to write a test
33 | * make sure to run `fmt.sh` and `test.sh` in the root of the repo to ensure that your code is
34 | properly formatted and that tests pass (NOTE: we integrate Travis with GitHub for continuous
35 | integration)
36 |
37 | ## Submitting Changes
38 |
39 | * push your changes to your branch in your fork of the repository
40 | * submit a pull request against nsqio's repository
41 | * comment in the pull request when you're ready for the changes to be reviewed: `"ready for review"`
42 |
--------------------------------------------------------------------------------
/nsqdserver/test/certs/cert.pem:
--------------------------------------------------------------------------------
1 | -----BEGIN CERTIFICATE-----
2 | MIIEbjCCA1agAwIBAgIJAK6x7y6AwBmLMA0GCSqGSIb3DQEBBQUAMIGAMQswCQYD
3 | VQQGEwJVUzERMA8GA1UECBMITmV3IFlvcmsxFjAUBgNVBAcTDU5ldyBZb3JrIENp
4 | dHkxDDAKBgNVBAoTA05TUTETMBEGA1UEAxMKdGVzdC5sb2NhbDEjMCEGCSqGSIb3
5 | DQEJARYUbXJlaWZlcnNvbkBnbWFpbC5jb20wHhcNMTMwNjI4MDA0MzQ4WhcNMTYw
6 | NDE3MDA0MzQ4WjCBgDELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMRYw
7 | FAYDVQQHEw1OZXcgWW9yayBDaXR5MQwwCgYDVQQKEwNOU1ExEzARBgNVBAMTCnRl
8 | c3QubG9jYWwxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMIIB
9 | IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnX0KB+svwy+yHU2qggz/EaGg
10 | craKShagKo+9M9y5HLM852ngk5c+t+tJJbx3N954Wr1FXBuGIv1ltU05rU4zhvBS
11 | 25tVP1UIEnT5pBt2TeetLkl199Y7fxh1hKmnwJMG3fy3VZdNXEndBombXMmtXpQY
12 | shuEJHKeUNDbQKz5X+GjEdkTPO/HY/VMHsxS23pbSimQozMg3hvLIdgv0aS3QECz
13 | ydZBgTPThy3uDtHIuCpxCwXd/vDF68ATlYgo3h3lh2vxNwM/pjklIUhzMh4XaKQF
14 | 7m3/0KbtUcXfy0QHueeuMr11E9MAFNyRN4xf9Fk1yB97KJ3PJBTC5WD/m1nW+QID
15 | AQABo4HoMIHlMB0GA1UdDgQWBBR3HMBws4lmYYSIgwoZsfW+bbgaMjCBtQYDVR0j
16 | BIGtMIGqgBR3HMBws4lmYYSIgwoZsfW+bbgaMqGBhqSBgzCBgDELMAkGA1UEBhMC
17 | VVMxETAPBgNVBAgTCE5ldyBZb3JrMRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MQww
18 | CgYDVQQKEwNOU1ExEzARBgNVBAMTCnRlc3QubG9jYWwxIzAhBgkqhkiG9w0BCQEW
19 | FG1yZWlmZXJzb25AZ21haWwuY29tggkArrHvLoDAGYswDAYDVR0TBAUwAwEB/zAN
20 | BgkqhkiG9w0BAQUFAAOCAQEANOYTbanW2iyV1v4oYpcM/y3TWcQKzSME8D2SGFZb
21 | dbMYU81hH3TTlQdvyeh3FAcdjhKE8Xi/RfNNjEslTBscdKXePGpZg6eXRNJzPP5K
22 | KZPf5u6tcpAeUOKrMqbGwbE+h2QixxG1EoVQtE421szsU2P7nHRTdHzKFRnOerfl
23 | Phm3NocR0P40Rv7WKdxpOvqc+XKf0onTruoVYoPWGpwcLixCG0zu4ZQ23/L/Dy18
24 | 4u70Hbq6O/6kq9FBFaDNp3IhiEdu2Cq6ZplU6bL9XDF27KIEErHwtuqBHVlMG+zB
25 | oH/k9vZvwH7OwAjHdKp+1yeZFLYC8K5hjFIHqcdwpZCNIg==
26 | -----END CERTIFICATE-----
27 |
--------------------------------------------------------------------------------
/nsqlookupd/tcp.go:
--------------------------------------------------------------------------------
1 | package nsqlookupd
2 |
3 | import (
4 | "io"
5 | "net"
6 | "time"
7 |
8 | "github.com/youzan/nsq/internal/protocol"
9 | )
10 |
11 | type tcpServer struct {
12 | ctx *Context
13 | }
14 |
15 | func (p *tcpServer) Handle(clientConn net.Conn) {
16 | nsqlookupLog.Logf("TCP: new client(%s)", clientConn.RemoteAddr())
17 |
18 | // The client should initialize itself by sending a 4 byte sequence indicating
19 | // the version of the protocol that it intends to communicate, this will allow us
20 | // to gracefully upgrade the protocol away from text/line oriented to whatever...
21 | buf := make([]byte, 4)
22 | clientConn.SetReadDeadline(time.Now().Add(time.Second * 3))
23 | _, err := io.ReadFull(clientConn, buf)
24 | if err != nil {
25 | nsqlookupLog.Logf(" failed to read protocol version - %s from client: %v", err, clientConn.RemoteAddr())
26 | clientConn.Close()
27 | return
28 | }
29 | protocolMagic := string(buf)
30 |
31 | nsqlookupLog.Logf("CLIENT(%s): desired protocol magic '%s'",
32 | clientConn.RemoteAddr(), protocolMagic)
33 |
34 | var prot protocol.Protocol
35 | switch protocolMagic {
36 | case " V1":
37 | prot = &LookupProtocolV1{ctx: p.ctx}
38 | default:
39 | protocol.SendResponse(clientConn, []byte("E_BAD_PROTOCOL"))
40 | clientConn.Close()
41 | nsqlookupLog.LogErrorf(" client(%s) bad protocol magic '%s'",
42 | clientConn.RemoteAddr(), protocolMagic)
43 | return
44 | }
45 |
46 | err = prot.IOLoop(clientConn)
47 | if err != nil {
48 | if err == io.EOF {
49 | nsqlookupLog.Logf(" client(%s) - %s", clientConn.RemoteAddr(), err)
50 | } else {
51 | nsqlookupLog.LogWarningf(" client(%s) - %s", clientConn.RemoteAddr(), err)
52 | }
53 | return
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/nsqdserver/tcp.go:
--------------------------------------------------------------------------------
1 | package nsqdserver
2 |
3 | import (
4 | "io"
5 | "net"
6 |
7 | "time"
8 |
9 | "github.com/youzan/nsq/internal/levellogger"
10 | "github.com/youzan/nsq/internal/protocol"
11 | "github.com/youzan/nsq/nsqd"
12 | )
13 |
14 | type tcpServer struct {
15 | ctx *context
16 | }
17 |
18 | func (p *tcpServer) Handle(clientConn net.Conn) {
19 | // The client should initialize itself by sending a 4 byte sequence indicating
20 | // the version of the protocol that it intends to communicate, this will allow us
21 | // to gracefully upgrade the protocol away from text/line oriented to whatever...
22 | clientConn.SetReadDeadline(time.Now().Add(p.ctx.getOpts().ClientTimeout))
23 | buf := make([]byte, 4)
24 | _, err := io.ReadFull(clientConn, buf)
25 | if err != nil {
26 | nsqd.NsqLogger().Logf(" failed to read protocol version - %s from client: %v", err, clientConn.RemoteAddr())
27 | clientConn.Close()
28 | return
29 | }
30 | protocolMagic := string(buf)
31 |
32 | if nsqd.NsqLogger().Level() >= levellogger.LOG_DEBUG {
33 | nsqd.NsqLogger().LogDebugf("new CLIENT(%s): desired protocol magic '%s'",
34 | clientConn.RemoteAddr(), protocolMagic)
35 | }
36 |
37 | var prot protocol.Protocol
38 | switch protocolMagic {
39 | case " V2":
40 | prot = &protocolV2{ctx: p.ctx}
41 | default:
42 | protocol.SendFramedResponse(clientConn, frameTypeError, []byte("E_BAD_PROTOCOL"))
43 | clientConn.Close()
44 | nsqd.NsqLogger().Logf("client(%s) bad protocol magic '%s'",
45 | clientConn.RemoteAddr(), protocolMagic)
46 | return
47 | }
48 |
49 | err = prot.IOLoop(clientConn)
50 | if err != nil {
51 | nsqd.NsqLogger().Logf("client(%s) error - %s", clientConn.RemoteAddr(), err)
52 | return
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/contrib/nsqlookupd.cfg.example:
--------------------------------------------------------------------------------
1 | ## : to listen on for TCP clients
2 | tcp_address = "0.0.0.0:4160"
3 |
4 | ## : to listen on for HTTP clients
5 | http_address = "0.0.0.0:4161"
6 |
7 | ## rpc port used for cluster communication
8 | rpc_port = "4260"
9 | ## address that will be registered with lookupd (defaults to the OS hostname)
10 | #broadcast_address = ""
11 | ## the network interface for broadcast, the ip will be detected automatically.
12 | # use this configure instead of broadcast_address to keep all the configure is the same
13 | broadcast_interface = "eth0"
14 |
15 | ## local reverse proxy port, basically used for collecting the stats
16 | # reverse_proxy_port = "4163"
17 |
18 | cluster_id = "test-nsq-cluster-dev-1"
19 | ## the etcd cluster ip list
20 | cluster_leadership_addresses = "http://127.0.0.1:2379"
21 | # cluster_leadership_username = "username"
22 | # cluster_leadership_password = "password"
23 | ## default value is "NSQMetaData"
24 | # cluster_leadership_root_dir = "etcd_root_dir"
25 |
26 | ## duration of time a producer will remain in the active list since its last ping
27 | inactive_producer_timeout = "100s"
28 | # should at least twice as the ping interval on nsqd
29 | nsqd_ping_timeout= "15s"
30 |
31 | ## duration of time a producer will remain tombstoned if registration remains
32 | tombstone_lifetime = "45s"
33 |
34 | ## the detail of the log, larger number means more details
35 | log_level = 2
36 |
37 | ## if empty, use the default flag value in glog
38 | log_dir = "./"
39 |
40 | ## the time period (in hour) that the balance is allowed.
41 | balance_interval = ["4", "5"]
42 |
43 | ## allow return topic as writable while no any channel under the topic
44 | allow_write_with_nochannels = true
45 |
--------------------------------------------------------------------------------
/nsqd/backend_queue.go:
--------------------------------------------------------------------------------
1 | package nsqd
2 |
3 | type BackendOffset int64
4 |
5 | type BackendQueueOffset interface {
6 | Offset() BackendOffset
7 | }
8 |
9 | type BackendQueueEnd interface {
10 | Offset() BackendOffset
11 | TotalMsgCnt() int64
12 | IsSame(BackendQueueEnd) bool
13 | }
14 |
15 | // BackendQueue represents the behavior for the secondary message
16 | // storage system
17 | type BackendQueue interface {
18 | Put([]byte) error
19 | ReadChan() chan []byte // this is expected to be an *unbuffered* channel
20 | Close() error
21 | Delete() error
22 | Depth() int64
23 | Empty() error
24 | }
25 |
26 | // for topic producer
27 | type BackendQueueWriter interface {
28 | Put([]byte) (BackendOffset, int32, int64, error)
29 | Close() error
30 | Delete() error
31 | Empty() error
32 | Flush(bool) error
33 | GetQueueWriteEnd() BackendQueueEnd
34 | GetQueueReadStart() BackendQueueEnd
35 | GetQueueReadEnd() BackendQueueEnd
36 | RollbackWrite(BackendOffset, uint64) error
37 | ResetWriteEnd(BackendOffset, int64) error
38 | }
39 |
40 | type ReadResult struct {
41 | Offset BackendOffset
42 | MovedSize BackendOffset
43 | CurCnt int64
44 | Data []byte
45 | Err error
46 | }
47 |
48 | // for channel consumer
49 | type BackendQueueReader interface {
50 | ConfirmRead(BackendOffset, int64) error
51 | ResetReadToConfirmed() (BackendQueueEnd, error)
52 | SkipReadToOffset(BackendOffset, int64) (BackendQueueEnd, error)
53 | SkipReadToEnd() (BackendQueueEnd, error)
54 | Close() error
55 | // left data to be read
56 | Depth() int64
57 | DepthSize() int64
58 | GetQueueReadEnd() BackendQueueEnd
59 | GetQueueConfirmed() BackendQueueEnd
60 | Delete() error
61 | UpdateQueueEnd(BackendQueueEnd, bool) (bool, error)
62 | TryReadOne() (ReadResult, bool)
63 | }
64 |
--------------------------------------------------------------------------------
/nsqdserver/test/certs/client.key:
--------------------------------------------------------------------------------
1 | -----BEGIN RSA PRIVATE KEY-----
2 | MIIEogIBAAKCAQEAnWwWQyfFijE8wkbRM6/osgJmvvlyakjXkwTM+IvFigQ2cI3D
3 | JybjvcaZEushBlSm+vge01+Gbsj5DX0i5OEG4+BRDih+kyAMLkSsF54y8EfrIVhD
4 | AB5hziI8LZB9KXIPGznWTMtOu0sZLMTVe/rxy6y6f5uPfArJYEYjut/HGQCxsJsl
5 | TkNstbE5tvVT2JJYSXoyAPu/ZnTA7hoAb1u5vOh7aB6q6uhA+w/wtzhDTzkdTNm/
6 | B3gK312/ePOotspJQ0MWopRft/2kfp3+fd7pYrk9hMVIziucOX+YFFPSblfYDHCS
7 | oxjBRBjviFmp8K/724qgKqZvDQIdgyMczcbDCwIDAQABAoIBAHIhXPqPKR6Jl/1a
8 | Dm4kyNXV+9vVSehYq0JBpsFDxJHNgBKWFLib7101UOFwat8GHJGztdWVWgeT2Uy4
9 | PsYlhzNK/DkBjtLn1yzsSMBOx6uA2K2yKdE6NbqXA34dVUDpGp6ojf/bHleXmMbm
10 | ZcifJRJ/Ri8h/yL1Rw5vi5P+jYBH52jfvujnjWUR+Deaz211WyDKT2mYmrRKsugd
11 | a6xvqndez3t7VCHwAFJTKYUAN9JTK2EGt8gwWYDmtpvRB3qlSzN+pzuSXA2NFO2V
12 | J2xjzvTYEOhIiAjae0YwnqID7wifIrWp1hX8EMm8Jji9yjGEZFnbA9MJD3unozFM
13 | f72GH1ECgYEAzGgzhXFl7iSASZEW6TEui9Le1OtMcMh5ZCmPpTRTorg8dzwEfjAN
14 | o2IzV3TTsa/XzUpcurHbz+fjXYIGV0Hj/MXWLKgkti9iJG+NIujdmi28lV8LHBZS
15 | fcyG6l8doRBEJMXEOaspbBkDIB8I7Cxe59y3UQoJs64KgtMllnz3a1cCgYEAxSfz
16 | 5NL9+1Nfvbg4RSyeBFtADzVuHLAp8IgQp+wq033e9aUcGw0B+mV0Cs2CbI/Qh+cQ
17 | QLIFx9pqJW8nfVZbXhqlDUSIVz1SX444MdVyyCPTjzMXnBQK94WCJsJrc6r2D0v+
18 | V3t+1xqHT8dCLZzBdZcZ7Kh6qrq0wfBE++aYCW0CgYBmFVw1tq5rQnt3QiipCVDv
19 | r75to52nbdeCOGdu4DUsvMol2il8Q89r2208rjHTObiS5CJvcsgOiwa3+mnkRhnw
20 | cXVHFCpRelSPUc6wso+JonHeP3sy6csE3JI8logbmh4OJ16aVmr3e35eGmHAYWB8
21 | d3Rt2o+B1teVApJxF7QRPQKBgAEm9Z7tuLxCRXn3XKTS2O2PlvU/y6/xO52DPg2t
22 | LKblwmrPFzX1QoCzHRV/E/cMdByHNioaoDwFw78KQkHSQmBvaW2FbAKOgi+6xgtM
23 | gwo2Y2Mf2lB4MYQ2zIbrdfNMGXoHaKocN6aMKiRGNtXLZ7oHMwiWB3l989Z6T/zk
24 | 4RiVAoGAPOZfwwqNkNRelQ52907gzRJT8DoGzNdxWdI+u/J8f2TBNHgwnn+Htibe
25 | O8m6qPQb0SbsGl4Ms+F77v1Dfk/VJa6CSFHavogyKg/FlyjIMIb4ptpzjngjDQ2T
26 | Ot2q4vt1Uchx24UL35FjwPzPST3WJDyKxv2d1yVp7sboCJJ2JUQ=
27 | -----END RSA PRIVATE KEY-----
28 |
--------------------------------------------------------------------------------
/nsqdserver/test/certs/key.pem:
--------------------------------------------------------------------------------
1 | -----BEGIN RSA PRIVATE KEY-----
2 | MIIEogIBAAKCAQEAnX0KB+svwy+yHU2qggz/EaGgcraKShagKo+9M9y5HLM852ng
3 | k5c+t+tJJbx3N954Wr1FXBuGIv1ltU05rU4zhvBS25tVP1UIEnT5pBt2TeetLkl1
4 | 99Y7fxh1hKmnwJMG3fy3VZdNXEndBombXMmtXpQYshuEJHKeUNDbQKz5X+GjEdkT
5 | PO/HY/VMHsxS23pbSimQozMg3hvLIdgv0aS3QECzydZBgTPThy3uDtHIuCpxCwXd
6 | /vDF68ATlYgo3h3lh2vxNwM/pjklIUhzMh4XaKQF7m3/0KbtUcXfy0QHueeuMr11
7 | E9MAFNyRN4xf9Fk1yB97KJ3PJBTC5WD/m1nW+QIDAQABAoIBACvtfKbIywG+hAf4
8 | ad7skRjx5DcbA2e29+XnQfb9UgTXWd2SgrmoLi5OypBkCTzkKN3mfTo70yZfV8dC
9 | Sxwz+9tfnTz0DssjhKThS+CiaFVCkeOfSfBfKSlCQUVHrSrh18CDhP+yvDlJwQTZ
10 | zSQMfPcsh9bmJe2kqtQP7ZgUp1o+vaB8Sju8YYrO6FllxbdLRGm4pfvvrHIRRmXa
11 | oVHn0ei0JpwoTY9kHYht4LNeJnbP/MCWdmcuv3Gnel7jAlhaKab5aNIGr0Xe7aIQ
12 | iX6mpZ0/Rnt8o/XcTOg8l3ruIdVuySX6SYn08JMnfFkXdNYRVhoV1tC5ElWkaZLf
13 | hPmj2yECgYEAyts0R0b8cZ6HTAyuLm3ilw0s0v0/MM9ZtaqMRilr2WEtAhF0GpHG
14 | TzmGnii0WcTNXD7NTsNcECR/0ZpXPRleMczsL2Juwd4FkQ37h7hdKPseJNrfyHRg
15 | VolOFBX9H14C3wMB9cwdsG4Egw7fE27WCoreEquHgwFxl1zBrXKH088CgYEAxr8w
16 | BKZs0bF7LRrFT5pH8hpMLYHMYk8ZIOfgmEGVBKDQCOERPR9a9kqUss7wl/98LVNK
17 | RnFlyWD6Z0/QcQsLL4LjBeZJ25qEMc6JXm9VGAzhXA1ZkUofVoYCnG+f6KUn8CuJ
18 | /AcV2ZDFsEP10IiQG0hKsceXiwFEvEr8306tMrcCgYBLgnscSR0xAeyk71dq6vZc
19 | ecgEpcX+2kAvclOSzlpZ6WVCjtKkDT0/Qk+M0eQIQkybGLl9pxS+4Yc+s2/jy2yX
20 | pwsHvGE0AvwZeZX2eDcdSRR4bYy9ZixyKdwJeAHnyivRbaIuJ5Opl9pQGpoI9snv
21 | 1K9DTdw8dK4exKVHdgl/WwKBgDkmLsuXg4EEtPOyV/xc08VVNIR9Z2T5c7NXmeiO
22 | KyiKiWeUOF3ID2L07S9BfENozq9F3PzGjMtMXJSqibiHwW6nB1rh7mj8VHjx9+Q0
23 | xVZGFeNfX1r84mgB3uxW2LeQDhzsmB/lda37CC14TU3qhu2hawEV8IijE73FHlOk
24 | Dv+fAoGAI4/XO5o5tNn5Djo8gHmGMCbinUE9+VySxl7wd7PK8w2VSofO88ofixDk
25 | NX94yBYhg5WZcLdPm45RyUnq+WVQYz9IKUrdxLFTH+wxyzUqZCW7jgXCvWV+071q
26 | vqm9C+kndq+18/1VKuCSGWnF7Ay4lbsgPXY2s4VKRxcb3QpZSPU=
27 | -----END RSA PRIVATE KEY-----
28 |
--------------------------------------------------------------------------------
/nsqdserver/test/certs/server.key:
--------------------------------------------------------------------------------
1 | -----BEGIN PRIVATE KEY-----
2 | MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDL2jtqe4l5v9iY
3 | JJYipX/QJRV/VKulznVUFZW1tBHqfgaZ8Nf/3Sn1nq3tDrDnfHa1fVuvy/VxE1Yg
4 | Jr9K7CmWEHxLpDa8WP4RoED5H7LM8TaJ++9Ph3+hm8d3+r11kK1DdaRmECUg7PCe
5 | mMzWKVxzw6qsDq2uZhaQbu9xhwJhIg4Uldud90RcgFBJS70BWMc5DvJK01nWy8D6
6 | 3gnb3SlSmeSDTCN7jC1oi4Ao94VPJCxlNj/3ql+aRHnOpvDal9INCQ4TB8XeH3pI
7 | W5pBpg39zufynZDJb9eG/1WSamd6dmWtwokQtFJOB6StbRkxAAkc/yVNYZAAs01H
8 | 8F4UipBpAgMBAAECggEBALq9ED6sKjR4dK5ynYHhf6MpxXaeFptN9JbxKvKHoP3h
9 | MqTkRsohTGWvVX5aPd3gW1JIIqx4tz89SmErfYLhsfrA4UvdgzKohx2YYsyTqBQr
10 | Rx3BxuhgpJ9jd6dF7zZ6MO4iH1ZLpedyhb2TFdl9jC6T6I2y6JFg/XyT0w55ccmN
11 | ZtKmKcA+WR68uOovuvEJv1SJSFaJdbOd1Z82UkuvLSzNuoImCrFTR1LlGxswKyqN
12 | 0cApMEh/f+rkTJ1XqaJ0lKMII2e52mPxThdfbjqDjjACL+SIh6mNiQAjrTEQ60DN
13 | dzhZB5JnSFSPBk/UDyzvNBR/4iMpOWvHhNhKZ1JpZH0CgYEA/ixMispEtkGAUnwI
14 | 7GXNgOJaDU3u79slyvmrOHNMa/n2fqOOw1bXlQJu+iHhPFpQvgb8DWdhdWhwxPIf
15 | en4T4tSuAojx7/EWN4eIAiPCWww/Bbo0UvP+mR/gBDU+vwNcc67D6lJJewEBEgTp
16 | KrgW21xtoTjdW8TBIwcXi2FDKZcCgYEAzVFWuZD1WP+/jpADB946hb3Xej5RgxK3
17 | F+nJ1O0nXSRl8g4dV3J4rkpZi0ADPnErX89sq7OmCdiReR0Amn/zJNOoc2tt05Tc
18 | 3tlhuJNvzdxn5h1f2MJy6KBAPGKh6knN3jPaMQmVKO0tntq8vkDEXbIWYXr7rOKE
19 | kZedCWFzVf8CgYAhaWNab0JfDvc2YJWvtaYxBhA+ZXs+TnmGWBGY5xcprn8noIp4
20 | xSarsNkzylZYX6rzf+mMPYXDMEp6qTVYHdCxZbHGJYhgOTnpbMiFN3wqCc8TCqos
21 | KWTTbiw9gV5RL7jsgBHC8LQ2Fii03K5l7jMa/OhwZBA9Kv0UhK8kpFF22wKBgQCv
22 | 3Nbi9ZYhmFY0v4fJCoGoItbZf12EXthzZC4b8tJ/xniH360kRQm3iDmwu7DLoFXd
23 | qCbV33f+AQGB8uSccbILPjyQOFmHHIAfZaV9WtQuSXa5NTnuyYdxWCoNTxKPQo5S
24 | oKlozWN0/crQwV1fvMw10R+4eYXvLMsOhgXdjhojcwKBgAMK+ax5BcG64CVBTqVt
25 | VU/pHC9pvS+7pjSySUaiiTtmMRPXg/Yrt6TCgA5JymAmdEro/7+5/s5p4WReNTpF
26 | yxSbkinQ5E4PxIDCwlEQKDz44G7wNOZ9mQn7aBwPlZAN2AJHvN0PXQsfuT+GVb8s
27 | IL4rkoOH4FXBDvqIgFYHe09D
28 | -----END PRIVATE KEY-----
29 |
--------------------------------------------------------------------------------
/consistence/nsqd_node_etcd_test.go:
--------------------------------------------------------------------------------
1 | package consistence
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "testing"
7 | "time"
8 |
9 | "github.com/youzan/nsq/internal/test"
10 | "golang.org/x/net/context"
11 | )
12 |
13 | func TestNodeRe(t *testing.T) {
14 | ClusterID := "test-nsq-cluster-unit-test-etcd-leadership"
15 | nodeMgr, _ := NewNsqdEtcdMgr(testEtcdServers, testEtcdUsername, testEtcdPassword)
16 | nodeMgr.InitClusterID(ClusterID)
17 | ID := "unit-test-etcd-node1"
18 | nodeInfo := &NsqdNodeInfo{
19 | ID: ID,
20 | NodeIP: "127.0.0.1",
21 | TcpPort: "2222",
22 | RpcPort: "2223",
23 | }
24 | err := nodeMgr.RegisterNsqd(nodeInfo)
25 | test.Nil(t, err)
26 | time.Sleep(10 * time.Second)
27 | err = nodeMgr.UnregisterNsqd(nodeInfo)
28 | test.Nil(t, err)
29 | }
30 |
31 | func TestETCDWatch(t *testing.T) {
32 | client, _ := NewEClient(testEtcdServers, testEtcdUsername, testEtcdPassword)
33 | watcher := client.Watch("q11", 0, true)
34 | ctx, cancel := context.WithCancel(context.Background())
35 | go func() {
36 | time.Sleep(10 * time.Second)
37 | cancel()
38 | }()
39 |
40 | for {
41 | rsp, err := watcher.Next(ctx)
42 | if err != nil {
43 | if err == context.Canceled {
44 | fmt.Println("watch canceled")
45 | return
46 | } else {
47 | time.Sleep(5 * time.Second)
48 | }
49 | continue
50 | }
51 | fmt.Println(rsp.Action, rsp.Node.Key, rsp.Node.Value)
52 | }
53 | }
54 |
55 | func TestEqualSesssion(t *testing.T) {
56 | var new TopicLeaderSession
57 | new.LeaderNode = &NsqdNodeInfo{}
58 | v, _ := json.Marshal(new)
59 | var old TopicLeaderSession
60 | json.Unmarshal(v, &old)
61 | if *new.LeaderNode != *old.LeaderNode {
62 | t.Errorf("should equal: %v, %v", new, old)
63 | }
64 | if !new.IsSame(&old) {
65 | t.Errorf("should equal: %v, %v", new, old)
66 | }
67 | }
68 |
--------------------------------------------------------------------------------
/nsqd/in_flight_pqueue_test.go:
--------------------------------------------------------------------------------
1 | package nsqd
2 |
3 | import (
4 | "fmt"
5 | "math/rand"
6 | "sort"
7 | "testing"
8 | )
9 |
10 | func TestPriorityQueue(t *testing.T) {
11 | c := 100
12 | pq := newInFlightPqueue(c)
13 |
14 | for i := 0; i < c+1; i++ {
15 | pq.Push(&Message{belongedConsumer: NewFakeConsumer(int64(i)), pri: int64(i)})
16 | }
17 | equal(t, len(pq), c+1)
18 | equal(t, cap(pq), c*2)
19 |
20 | for i := 0; i < c+1; i++ {
21 | msg := pq.Pop()
22 | equal(t, msg.GetClientID(), int64(i))
23 | }
24 | equal(t, cap(pq), c/4)
25 | }
26 |
27 | func TestUnsortedInsert(t *testing.T) {
28 | c := 100
29 | pq := newInFlightPqueue(c)
30 | ints := make([]int, 0, c)
31 |
32 | for i := 0; i < c; i++ {
33 | v := rand.Int()
34 | ints = append(ints, v)
35 | pq.Push(&Message{pri: int64(v)})
36 | }
37 | equal(t, len(pq), c)
38 | equal(t, cap(pq), c)
39 |
40 | sort.Sort(sort.IntSlice(ints))
41 |
42 | for i := 0; i < c; i++ {
43 | msg, _ := pq.PeekAndShift(int64(ints[len(ints)-1]))
44 | equal(t, msg.pri, int64(ints[i]))
45 | }
46 | }
47 |
48 | func TestRemove(t *testing.T) {
49 | c := 100
50 | pq := newInFlightPqueue(c)
51 |
52 | msgs := make(map[MessageID]*Message)
53 | for i := 0; i < c; i++ {
54 | m := &Message{pri: int64(rand.Intn(100000000))}
55 | m.ID = MessageID(m.pri)
56 | msgs[m.ID] = m
57 | pq.Push(m)
58 | }
59 |
60 | for i := 0; i < 10; i++ {
61 | idx := rand.Intn((c - 1) - i)
62 | var fm *Message
63 | for _, m := range msgs {
64 | if m.index == idx {
65 | fm = m
66 | break
67 | }
68 | }
69 | rm := pq.Remove(idx)
70 | equal(t, fmt.Sprintf("%v", fm.ID), fmt.Sprintf("%v", rm.ID))
71 | }
72 |
73 | lastPriority := pq.Pop().pri
74 | for i := 0; i < (c - 10 - 1); i++ {
75 | msg := pq.Pop()
76 | equal(t, lastPriority <= msg.pri, true)
77 | lastPriority = msg.pri
78 | }
79 | }
80 |
--------------------------------------------------------------------------------
/internal/test/assertions.go:
--------------------------------------------------------------------------------
1 | package test
2 |
3 | import (
4 | "path/filepath"
5 | "reflect"
6 | "runtime"
7 | "testing"
8 | )
9 |
10 | func Assert(t *testing.T, condition bool, msg string, v ...interface{}) {
11 | if !condition {
12 | _, file, line, _ := runtime.Caller(1)
13 | t.Logf("\033[31m%s:%d: "+msg+"\033[39m\n\n",
14 | append([]interface{}{filepath.Base(file), line}, v...)...)
15 | t.FailNow()
16 | }
17 | }
18 |
19 | func Equal(t *testing.T, expected, actual interface{}) {
20 | if !reflect.DeepEqual(expected, actual) {
21 | _, file, line, _ := runtime.Caller(1)
22 | t.Logf("\033[31m%s:%d:\n\n\t %#v (expected)\n\n\t!= %#v (actual)\033[39m\n\n",
23 | filepath.Base(file), line, expected, actual)
24 | t.FailNow()
25 | }
26 | }
27 |
28 | func NotEqual(t *testing.T, expected, actual interface{}) {
29 | if reflect.DeepEqual(expected, actual) {
30 | _, file, line, _ := runtime.Caller(1)
31 | t.Logf("\033[31m%s:%d:\n\n\tvalue should not equal %#v\033[39m\n\n",
32 | filepath.Base(file), line, actual)
33 | t.FailNow()
34 | }
35 | }
36 |
37 | func Nil(t *testing.T, object interface{}) {
38 | if !isNil(object) {
39 | _, file, line, _ := runtime.Caller(1)
40 | t.Logf("\033[31m%s:%d:\n\n\t (expected)\n\n\t!= %#v (actual)\033[39m\n\n",
41 | filepath.Base(file), line, object)
42 | t.FailNow()
43 | }
44 | }
45 |
46 | func NotNil(t *testing.T, object interface{}) {
47 | if isNil(object) {
48 | _, file, line, _ := runtime.Caller(1)
49 | t.Logf("\033[31m%s:%d:\n\n\tExpected value not to be \033[39m\n\n",
50 | filepath.Base(file), line)
51 | t.FailNow()
52 | }
53 | }
54 |
55 | func isNil(object interface{}) bool {
56 | if object == nil {
57 | return true
58 | }
59 |
60 | value := reflect.ValueOf(object)
61 | kind := value.Kind()
62 | if kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() {
63 | return true
64 | }
65 |
66 | return false
67 | }
68 |
--------------------------------------------------------------------------------
/nsqadmin/static/js/router.js:
--------------------------------------------------------------------------------
1 | var Backbone = require('backbone');
2 |
3 | var Pubsub = require('./lib/pubsub');
4 |
5 |
6 | var Router = Backbone.Router.extend({
7 | routes: {
8 | '': 'topics',
9 | 'topics/(:topic)(/:channel)': 'topic',
10 | 'lookup': 'lookup',
11 | 'nodes(/:node)': 'nodes',
12 | 'counter': 'counter',
13 | 'statistics(/:filter)': 'statistics',
14 | 'search': 'search'
15 | },
16 |
17 | defaultRoute: 'topics',
18 |
19 | initialize: function() {
20 | this.currentRoute = this.defaultRoute;
21 | this.listenTo(this, 'route', function(route, params) {
22 | this.currentRoute = route || this.defaultRoute;
23 | // console.log('Route: %o; params: %o', route, params);
24 | });
25 | },
26 |
27 | start: function() {
28 | Backbone.history.start({
29 | 'pushState': true
30 | });
31 | },
32 |
33 | topics: function() {
34 | Pubsub.trigger('topics:show');
35 | },
36 |
37 | topic: function(topic, channel) {
38 | if (channel !== null) {
39 | Pubsub.trigger('channel:show', topic, channel);
40 | return;
41 | }
42 | Pubsub.trigger('topic:show', topic);
43 | },
44 |
45 | lookup: function() {
46 | Pubsub.trigger('lookup:show');
47 | },
48 |
49 | nodes: function(node) {
50 | if (node !== null) {
51 | Pubsub.trigger('node:show', node);
52 | return;
53 | }
54 | Pubsub.trigger('nodes:show');
55 | },
56 |
57 | counter: function() {
58 | Pubsub.trigger('counter:show');
59 | },
60 |
61 | search: function() {
62 | Pubsub.trigger('search:show');
63 | },
64 |
65 | statistics: function() {
66 | Pubsub.trigger('statistics:show');
67 | }
68 | });
69 |
70 |
71 | module.exports = new Router();
72 |
--------------------------------------------------------------------------------
/nsqadmin/static/js/app_state.js:
--------------------------------------------------------------------------------
1 | var Backbone = require('backbone');
2 | var _ = require('underscore');
3 |
4 | var AppState = Backbone.Model.extend({
5 | defaults: function() {
6 | return {
7 | 'VERSION': VERSION,
8 | 'GRAPHITE_URL': GRAPHITE_URL,
9 | 'GRAPH_ENABLED': GRAPH_ENABLED,
10 | 'STATSD_INTERVAL': STATSD_INTERVAL,
11 | 'USE_STATSD_PREFIXES': USE_STATSD_PREFIXES,
12 | 'STATSD_COUNTER_FORMAT': STATSD_COUNTER_FORMAT,
13 | 'STATSD_GAUGE_FORMAT': STATSD_GAUGE_FORMAT,
14 | 'STATSD_PREFIX': STATSD_PREFIX,
15 | 'NSQLOOKUPD': NSQLOOKUPD,
16 | 'ALLNSQLOOKUPDS': ALLNSQLOOKUPDS,
17 | 'DCNSQLOOKUPD' : DCNSQLOOKUPD,
18 | 'DCALLNSQLOOKUPDS': DCALLNSQLOOKUPDS,
19 | 'graph_interval': '2h',
20 | 'AUTH_URL' : AUTH_URL,
21 | 'LOGOUT_URL' : LOGOUT_URL,
22 | 'LOGIN' : LOGIN,
23 | 'USER' : USER,
24 | 'AUTH_ENABLED' : AUTH_ENABLED,
25 | 'HAS_ENDPOINT' : HAS_ENDPOINT,
26 | 'ENABLE_ZAN_TEST_SKIP' : ENABLE_ZAN_TEST_SKIP,
27 | };
28 | },
29 |
30 | initialize: function() {
31 | this.on('change:graph_interval', function(model, v) {
32 | localStorage.setItem('graph_interval', v);
33 | });
34 |
35 | var qp = _.object(_.compact(_.map(window.location.search.slice(1).split('&'),
36 | function(item) { if (item) { return item.split('='); } })));
37 |
38 | var def = this.get('GRAPH_ENABLED') ? '2h' : 'off';
39 | var interval = qp['t'] || localStorage.getItem('graph_interval') || def;
40 | this.set('graph_interval', interval);
41 | },
42 |
43 | url: function(url) {
44 | return '/api' + url;
45 | }
46 | });
47 |
48 | var appState = new AppState();
49 |
50 | window.AppState = appState;
51 |
52 | module.exports = appState;
53 |
--------------------------------------------------------------------------------
/nsqdserver/test/certs/ca.key:
--------------------------------------------------------------------------------
1 | -----BEGIN ENCRYPTED PRIVATE KEY-----
2 | MIIFHzBJBgkqhkiG9w0BBQ0wPDAbBgkqhkiG9w0BBQwwDgQIOnMHjZ/OtboCAggA
3 | MB0GCWCGSAFlAwQBKgQQraghngsT2Jfh2Jl4Bs8LrASCBNAbx/YlZvCgcV9zSkLw
4 | 8kBFtB8+Livnv5BoI0S2p3XddAManDoErdRh2LoAOtNTCDEuiJ3oCHy/DZ1eaDhv
5 | qXJ4fc2JfA5qIMOHmFiCE3fbVoy47wurggcBw7tLAKoAg+Jo2R9j7k++rZeKn6p5
6 | KBHbZpfQyVLczJktwm2l/LLc85tGfhQvgWlnPHg2GFhETmwNvGxGruyelPJizHG0
7 | 6wmAn9eOMycM9YrmSQ6VfMtxza110nPCk0yyjqnDnRgc8wFmMohbXUxnz3HZszhb
8 | /2TRcYACS4Lk0/u0LRWFMLH1IfWd0L0SASL1oDXzMPVCp5vbGSIThZAJGe4ZFel3
9 | rp56A2dX+aXxzZ1FK8CaPsZpHCyv7j/4wxW1M186Yh32QfyhT8tT2KLY2lSepdN2
10 | bPHcXq7hkfJ4xMmWt5qlzOSeDDt3/Z2pHQaQ/Add+JtFZnfvY3QodwnPTe9dQugi
11 | bGQ/UA0SvcWnyLRBznP2fibHAWN/NjNBcMKH9UX8eE4zMx+rbOXaPUqcSkkGquGT
12 | GlBQEVe+9GqbhHRENylgGflDH0PhWQOTOjeFeVKTRA08Zrc2nM55MJgPoou4RE+3
13 | LnFajLFSgUYQOTqkfrOAnNdtS6Aww6agRZYWF3VhGFhK5A9kmwaB0qAr8YVFOhiV
14 | q+BW5bQ72y9jnhqe8MRJM0nrVXmbjd66WIa0jyHpDWoosKF3CvTvQYgrw++p5QS7
15 | b3TVeL0+Gf0EuxD581WVkQz8NK2SnwGIaXN9Aw6bSsKGsUbJwsdThrN4q8D2OPeH
16 | gMCPRmFfFloRdqtK7O7MdtOZ4f7cQ1CpEDLlHNXT/8ULEZJWB1lFy75k9NAzG2Cv
17 | 0fYbX6pIRX90WmfuwXRfHz4ycD/40QCRk+662YGYWXPuJu+anNugzmvqP40QYx3X
18 | cvHk1t0JnwfSUUROzUE38i4VhNffG4mAE5N7mJkmCPr0fDiVu+pOLJkICxi2k4iV
19 | /6qt4XOlWxBAkKnd7McubMvhrZXQEQl2kC/Z0DvJWz+9upkC5KiCrWonr1bwfPWz
20 | 6TV1p2A41fGLTr0UFmYihHLab1G/a+xRqwKpZDK9mVpc9RitOYwH5xaV/rG1h1ea
21 | arH5hKXm33YLBZpyGdquFp23pExtfamLcGpbh6W+iTFeOi5VwT5a7HIHcH2I2j9k
22 | wH1Kb2q0sw2tfmvMV43BsS3JVjpIhrVHEhbSD0ZzmDesjakRwF19X0J9drB0nYO3
23 | Z58uozNLZv1tIp1N4jCma5qirBrkwD8xnq4NdKAKJUlzgDt3RJAbCZMstjlnk+/J
24 | iVXAeeoTlioPnBbga/8ZfN2EUe+EQkuzW1z6ZXZBiYf0++MGCiYYnabOapwKBu08
25 | W3b4/ok5TQlaFeNeBjqSfoKRW6IquwYk0IollBIV2JQ28hIfzr2QwLxESz7CaJa/
26 | X6qK1RXummwVvoY8rdHHgBmSW29Hu0MEhjlB87UM++U9gD/mQjxp8k4Hb5NfBgjz
27 | UkrPM83JE4wThaJ0WLIYeToHaTJbT8ZyVVgNS2aUK/dWurC35/lhbkC35n7/Z+i4
28 | msUUpETqkQlxffy/Kd9Vc+DfIOYVON/j+kazPGV7xj2q/FxdesmpqSMVdcyR8DQ9
29 | uMgajFUlQJVi0YglX3L5oe0XWQ==
30 | -----END ENCRYPTED PRIVATE KEY-----
31 |
--------------------------------------------------------------------------------
/nsqadmin/static/js/models/rank.js:
--------------------------------------------------------------------------------
1 | var _ = require('underscore');
2 |
3 | var AppState = require('../app_state');
4 | var Backbone = require('backbone');
5 |
6 | var Rank = Backbone.Model.extend({
7 | idAttribute: 'filter',
8 |
9 | constructor: function Rank() {
10 | Backbone.Model.prototype.constructor.apply(this, arguments);
11 | },
12 |
13 | url: function() {
14 | return AppState.url('/statistics/' + encodeURIComponent(this.get('filter')));
15 | },
16 |
17 | parse: function(resp) {
18 | var filter = this.get('filter');
19 | resp['top10'] = _.map(resp['top10'], function(data) {
20 | switch (filter) {
21 | case 'channel-depth':
22 | data['rank_value'] = data['total_channel_depth'];
23 | break;
24 | case 'hourly-pubsize':
25 | data['rank_value'] = data['hourly_pubsize'];
26 | break;
27 | case 'channel-requeue':
28 | data['name'] = data['name'].replace(':', '/');
29 | data['rank_value'] = data['requeue_count'];
30 | break;
31 | case 'channel-timeout':
32 | data['name'] = data['name'].replace(':', '/');
33 | data['rank_value'] = data['timeout_count'];
34 | break;
35 | case 'channel-delayedqueue':
36 | data['name'] = data['name'].replace(':', '/');
37 | data['rank_value'] = data['delayed_queue_count'];
38 | break;
39 | default:
40 | data['rank_value'] = data['message_count'];
41 | }
42 | return data;
43 | });
44 | resp['top10'] = _.filter(resp['top10'], function(data) {
45 | return (null !== data['rank_value'] && data['rank_value'] > 0);
46 | });
47 | return resp;
48 | }
49 | });
50 |
51 | module.exports = Rank;
--------------------------------------------------------------------------------
/nsqd/buffer_pool.go:
--------------------------------------------------------------------------------
1 | package nsqd
2 |
3 | import (
4 | "bufio"
5 | "bytes"
6 | "io"
7 | "sync"
8 | )
9 |
10 | var bp sync.Pool
11 | var peekPool sync.Pool
12 |
13 | func init() {
14 | bp.New = func() interface{} {
15 | return &bytes.Buffer{}
16 | }
17 | peekPool.New = func() interface{} {
18 | return make([]Message, MaxWaitingDelayed)
19 | }
20 | }
21 |
22 | func bufferPoolGet() *bytes.Buffer {
23 | return bp.Get().(*bytes.Buffer)
24 | }
25 |
26 | func bufferPoolPut(b *bytes.Buffer) {
27 | b.Reset()
28 | bp.Put(b)
29 | }
30 |
31 | func peekBufPoolGet() []Message {
32 | return peekPool.Get().([]Message)
33 | }
34 |
35 | func peekBufPoolPut(b []Message) {
36 | peekPool.Put(b)
37 | }
38 |
39 | var (
40 | bufioReaderPool sync.Pool
41 | bufioWriter2kPool sync.Pool
42 | bufioWriter4kPool sync.Pool
43 | bufioWriter8kPool sync.Pool
44 | bufioSmallWriterPool sync.Pool
45 | )
46 |
47 | func bufioWriterPool(size int) *sync.Pool {
48 | switch size {
49 | case 100:
50 | return &bufioSmallWriterPool
51 | case 2 << 10:
52 | return &bufioWriter2kPool
53 | case 4 << 10:
54 | return &bufioWriter4kPool
55 | case 8 << 10:
56 | return &bufioWriter8kPool
57 | }
58 | return nil
59 | }
60 |
61 | func NewBufioReader(r io.Reader) *bufio.Reader {
62 | if v := bufioReaderPool.Get(); v != nil {
63 | br := v.(*bufio.Reader)
64 | br.Reset(r)
65 | return br
66 | }
67 | return bufio.NewReader(r)
68 | }
69 |
70 | func PutBufioReader(br *bufio.Reader) {
71 | br.Reset(nil)
72 | bufioReaderPool.Put(br)
73 | }
74 |
75 | func newBufioWriterSize(w io.Writer, size int) *bufio.Writer {
76 | pool := bufioWriterPool(size)
77 | if pool != nil {
78 | if v := pool.Get(); v != nil {
79 | bw := v.(*bufio.Writer)
80 | bw.Reset(w)
81 | return bw
82 | }
83 | }
84 | return bufio.NewWriterSize(w, size)
85 | }
86 |
87 | func putBufioWriter(bw *bufio.Writer) {
88 | bw.Reset(nil)
89 | if pool := bufioWriterPool(bw.Available()); pool != nil {
90 | pool.Put(bw)
91 | }
92 | }
93 |
--------------------------------------------------------------------------------
/internal/test/fakes.go:
--------------------------------------------------------------------------------
1 | package test
2 |
3 | import (
4 | "net"
5 | "time"
6 | )
7 |
8 | type FakeNetConn struct {
9 | ReadFunc func([]byte) (int, error)
10 | WriteFunc func([]byte) (int, error)
11 | CloseFunc func() error
12 | LocalAddrFunc func() net.Addr
13 | RemoteAddrFunc func() net.Addr
14 | SetDeadlineFunc func(time.Time) error
15 | SetReadDeadlineFunc func(time.Time) error
16 | SetWriteDeadlineFunc func(time.Time) error
17 | }
18 |
19 | func (f FakeNetConn) Read(b []byte) (int, error) { return f.ReadFunc(b) }
20 | func (f FakeNetConn) Write(b []byte) (int, error) { return f.WriteFunc(b) }
21 | func (f FakeNetConn) Close() error { return f.CloseFunc() }
22 | func (f FakeNetConn) LocalAddr() net.Addr { return f.LocalAddrFunc() }
23 | func (f FakeNetConn) RemoteAddr() net.Addr { return f.RemoteAddrFunc() }
24 | func (f FakeNetConn) SetDeadline(t time.Time) error { return f.SetDeadlineFunc(t) }
25 | func (f FakeNetConn) SetReadDeadline(t time.Time) error { return f.SetReadDeadlineFunc(t) }
26 | func (f FakeNetConn) SetWriteDeadline(t time.Time) error { return f.SetWriteDeadlineFunc(t) }
27 |
28 | type fakeNetAddr struct{}
29 |
30 | func (fakeNetAddr) Network() string { return "" }
31 | func (fakeNetAddr) String() string { return "" }
32 |
33 | func NewFakeNetConn() FakeNetConn {
34 | netAddr := fakeNetAddr{}
35 | return FakeNetConn{
36 | ReadFunc: func(b []byte) (int, error) { return 0, nil },
37 | WriteFunc: func(b []byte) (int, error) { return len(b), nil },
38 | CloseFunc: func() error { return nil },
39 | LocalAddrFunc: func() net.Addr { return netAddr },
40 | RemoteAddrFunc: func() net.Addr { return netAddr },
41 | SetDeadlineFunc: func(time.Time) error { return nil },
42 | SetWriteDeadlineFunc: func(time.Time) error { return nil },
43 | SetReadDeadlineFunc: func(time.Time) error { return nil },
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/apps/nsq_to_file/strftime.go:
--------------------------------------------------------------------------------
1 | // COPIED FROM https://github.com/jehiah/go-strftime
2 | package main
3 |
4 | import (
5 | "time"
6 | )
7 |
8 | // taken from time/format.go
9 | var conversion = map[string]string{
10 | /*stdLongMonth */ "B": "January",
11 | /*stdMonth */ "b": "Jan",
12 | // stdNumMonth */ "m": "1",
13 | /*stdZeroMonth */ "m": "01",
14 | /*stdLongWeekDay */ "A": "Monday",
15 | /*stdWeekDay */ "a": "Mon",
16 | // stdDay */ "d": "2",
17 | // stdUnderDay */ "d": "_2",
18 | /*stdZeroDay */ "d": "02",
19 | /*stdHour */ "H": "15",
20 | // stdHour12 */ "I": "3",
21 | /*stdZeroHour12 */ "I": "03",
22 | // stdMinute */ "M": "4",
23 | /*stdZeroMinute */ "M": "04",
24 | // stdSecond */ "S": "5",
25 | /*stdZeroSecond */ "S": "05",
26 | /*stdLongYear */ "Y": "2006",
27 | /*stdYear */ "y": "06",
28 | /*stdPM */ "p": "PM",
29 | // stdpm */ "p": "pm",
30 | /*stdTZ */ "Z": "MST",
31 | // stdISO8601TZ */ "z": "Z0700", // prints Z for UTC
32 | // stdISO8601ColonTZ */ "z": "Z07:00", // prints Z for UTC
33 | /*stdNumTZ */ "z": "-0700", // always numeric
34 | // stdNumShortTZ */ "b": "-07", // always numeric
35 | // stdNumColonTZ */ "b": "-07:00", // always numeric
36 | "%": "%",
37 | }
38 |
39 | // This is an alternative to time.Format because no one knows
40 | // what date 040305 is supposed to create when used as a 'layout' string
41 | // this takes standard strftime format options. For a complete list
42 | // of format options see http://strftime.org/
43 | func strftime(format string, t time.Time) string {
44 | layout := ""
45 | length := len(format)
46 | for i := 0; i < length; i++ {
47 | if format[i] == '%' && i <= length-2 {
48 | if layoutCmd, ok := conversion[format[i+1:i+2]]; ok {
49 | layout = layout + layoutCmd
50 | i++
51 | continue
52 | }
53 | }
54 | layout = layout + format[i:i+1]
55 | }
56 | return t.Format(layout)
57 | }
58 |
--------------------------------------------------------------------------------
/internal/http_api/topic_channel_args.go:
--------------------------------------------------------------------------------
1 | package http_api
2 |
3 | import (
4 | "errors"
5 |
6 | "github.com/youzan/nsq/internal/protocol"
7 | "strconv"
8 | )
9 |
10 | type getter interface {
11 | Get(key string) string
12 | }
13 |
14 | func GetTopicArg(rp getter) (string, error) {
15 | topicName := rp.Get("topic")
16 | if topicName == "" {
17 | return "", errors.New("MISSING_ARG_TOPIC")
18 | }
19 |
20 | if !protocol.IsValidTopicName(topicName) {
21 | return "", errors.New("INVALID_ARG_TOPIC")
22 | }
23 |
24 | return topicName, nil
25 | }
26 |
27 | func GetTopicChannelArgs(rp getter) (string, string, error) {
28 | topicName, err := GetTopicArg(rp)
29 | if err != nil {
30 | return "", "", err
31 | }
32 |
33 | channelName := rp.Get("channel")
34 | if channelName == "" {
35 | return "", "", errors.New("MISSING_ARG_CHANNEL")
36 | }
37 |
38 | if !protocol.IsValidChannelName(channelName) {
39 | return "", "", errors.New("INVALID_ARG_CHANNEL")
40 | }
41 |
42 | return topicName, channelName, nil
43 | }
44 |
45 | // partition can be missing, default as 0.
46 | func GetTopicPartitionArgs(rp getter) (string, int, error) {
47 | topicName, err := GetTopicArg(rp)
48 | if err != nil {
49 | return "", -1, err
50 | }
51 |
52 | topicPartStr := rp.Get("partition")
53 | topicPart := -1
54 | if topicPartStr == "" {
55 | topicPart = -1
56 | } else {
57 | var err error
58 | topicPart, err = strconv.Atoi(topicPartStr)
59 | if err != nil {
60 | return "", -1, err
61 | }
62 | }
63 | return topicName, topicPart, nil
64 | }
65 |
66 | func GetTopicPartitionChannelArgs(rp getter) (string, int, string, error) {
67 | topicName, topicPart, err := GetTopicPartitionArgs(rp)
68 | if err != nil {
69 | return "", -1, "", err
70 | }
71 | channelName := rp.Get("channel")
72 | if channelName == "" {
73 | return "", -1, "", errors.New("MISSING_ARG_CHANNEL")
74 | }
75 |
76 | if !protocol.IsValidChannelName(channelName) {
77 | return "", -1, "", errors.New("INVALID_ARG_CHANNEL")
78 | }
79 |
80 | return topicName, topicPart, channelName, nil
81 | }
82 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | PREFIX=/usr/local
2 | DESTDIR=
3 | PROJECT?=github.com/youzan/nsq
4 | BINDIR=${PREFIX}/bin
5 | COMMIT?=$(shell git rev-parse --short HEAD)
6 | BUILD_TIME?=$(shell date '+%Y-%m-%d_%H:%M:%S-%Z')
7 | GOFLAGS=-ldflags "-X ${PROJECT}/internal/version.Commit=${COMMIT} -X ${PROJECT}/internal/version.BuildTime=${BUILD_TIME}"
8 |
9 | BLDDIR = build
10 | EXT=
11 | ifeq (${GOOS},windows)
12 | EXT=.exe
13 | endif
14 |
15 | APPS = nsqd nsqlookupd nsqadmin nsq_pubsub nsq_to_nsq nsq_to_file nsq_to_http nsq_tail nsq_stat to_nsq nsq_data_tool nsqlookupd_migrate_proxy
16 | all: $(APPS)
17 |
18 | $(BLDDIR)/nsqd: $(wildcard apps/nsqd/*.go nsqd/*.go nsqdserver/*.go consistence/*.go internal/*/*.go)
19 | $(BLDDIR)/nsqlookupd: $(wildcard apps/nsqlookupd/*.go nsqlookupd/*.go consistence/*.go internal/*/*.go)
20 | $(BLDDIR)/nsqadmin: $(wildcard apps/nsqadmin/*.go nsqadmin/*.go nsqadmin/templates/*.go internal/*/*.go)
21 | $(BLDDIR)/nsq_pubsub: $(wildcard apps/nsq_pubsub/*.go internal/*/*.go)
22 | $(BLDDIR)/nsq_to_nsq: $(wildcard apps/nsq_to_nsq/*.go internal/*/*.go)
23 | $(BLDDIR)/nsq_to_file: $(wildcard apps/nsq_to_file/*.go internal/*/*.go)
24 | $(BLDDIR)/nsq_to_http: $(wildcard apps/nsq_to_http/*.go internal/*/*.go)
25 | $(BLDDIR)/nsq_tail: $(wildcard apps/nsq_tail/*.go internal/*/*.go)
26 | $(BLDDIR)/nsq_stat: $(wildcard apps/nsq_stat/*.go internal/*/*.go)
27 | $(BLDDIR)/to_nsq: $(wildcard apps/to_nsq/*.go internal/*/*.go)
28 | $(BLDDIR)/nsq_data_tool: $(wildcard apps/nsq_data_tool/*.go consistence/*.go nsqd/*.go internal/*/*.go)
29 | $(BLDDIR)/nsqlookupd_migrate_proxy: $(wildcard apps/nsqlookupd_migrate_proxy/*.go nsqlookupd_migrate/*.go)
30 |
31 |
32 | $(BLDDIR)/%:
33 | @mkdir -p $(dir $@)
34 | go build ${GOFLAGS} -o $@ ./apps/$*
35 |
36 | $(APPS): %: $(BLDDIR)/%
37 |
38 | clean:
39 | rm -fr $(BLDDIR)
40 |
41 | .PHONY: install clean all
42 | .PHONY: $(APPS)
43 |
44 | install: $(APPS)
45 | install -m 755 -d ${DESTDIR}${BINDIR}
46 | for APP in $^ ; do install -m 755 ${BLDDIR}/$$APP ${DESTDIR}${BINDIR}/$$APP${EXT} ; done
47 |
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Contributor Code of Conduct
2 |
3 | As contributors and maintainers of this project, and in the interest of fostering an open and
4 | welcoming community, we pledge to respect all people who contribute through reporting issues,
5 | posting feature requests, updating documentation, submitting pull requests or patches, and other
6 | activities.
7 |
8 | We are committed to making participation in this project a harassment-free experience for everyone,
9 | regardless of level of experience, gender, gender identity and expression, sexual orientation,
10 | disability, personal appearance, body size, race, ethnicity, age, religion, or nationality.
11 |
12 | Examples of unacceptable behavior by participants include:
13 |
14 | * The use of sexualized language or imagery
15 | * Personal attacks
16 | * Trolling or insulting/derogatory comments
17 | * Public or private harassment
18 | * Publishing other's private information, such as physical or electronic addresses, without explicit permission
19 | * Other unethical or unprofessional conduct.
20 |
21 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits,
22 | code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. By
23 | adopting this Code of Conduct, project maintainers commit themselves to fairly and consistently
24 | applying these principles to every aspect of managing this project. Project maintainers who do not
25 | follow or enforce the Code of Conduct may be permanently removed from the project team.
26 |
27 | This code of conduct applies both within project spaces and in public spaces when an individual is
28 | representing the project or its community.
29 |
30 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an
31 | issue or contacting one or more of the project maintainers.
32 |
33 | This Code of Conduct is adapted from the [Contributor Covenant][1], version 1.2.0, available at
34 | [http://contributor-covenant.org/version/1/2/0/][2].
35 |
36 | [1]: http://contributor-covenant.org
37 | [2]: http://contributor-covenant.org/version/1/2/0/
38 |
--------------------------------------------------------------------------------
/nsqadmin/static/html/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | nsqadmin
5 |
17 |
18 |
40 |
41 |
42 |
43 |
44 |
--------------------------------------------------------------------------------
/nsqlookupd/options.go:
--------------------------------------------------------------------------------
1 | package nsqlookupd
2 |
3 | import (
4 | "log"
5 | "os"
6 | "time"
7 |
8 | "github.com/youzan/nsq/internal/levellogger"
9 | )
10 |
11 | type Options struct {
12 | Verbose bool `flag:"verbose"`
13 |
14 | TCPAddress string `flag:"tcp-address"`
15 | HTTPAddress string `flag:"http-address"`
16 | MetricAddress string `flag:"metric-address"`
17 | RPCPort string `flag:"rpc-port"`
18 | BroadcastAddress string `flag:"broadcast-address"`
19 | BroadcastInterface string `flag:"broadcast-interface"`
20 |
21 | ReverseProxyPort string `flag:"reverse-proxy-port"`
22 |
23 | ClusterID string `flag:"cluster-id"`
24 | ClusterLeadershipAddresses string `flag:"cluster-leadership-addresses" cfg:"cluster_leadership_addresses"`
25 | ClusterLeadershipUsername string `flag:"cluster-leadership-username" cfg:"cluster_leadership_username"`
26 | ClusterLeadershipPassword string `flag:"cluster-leadership-password" cfg:"cluster_leadership_password"`
27 | ClusterLeadershipRootDir string `flag:"cluster-leadership-root-dir" cfg:"cluster_leadership_root_dir"`
28 |
29 | InactiveProducerTimeout time.Duration `flag:"inactive-producer-timeout"`
30 | NsqdPingTimeout time.Duration `flag:"nsqd-ping-timeout"`
31 | BalanceInterval []string `flag:"balance-interval"`
32 | AllowWriteWithNoChannels bool `flag:"allow-write-with-nochannels"`
33 |
34 | LogLevel int32 `flag:"log-level" cfg:"log_level"`
35 | LogDir string `flag:"log-dir" cfg:"log_dir"`
36 | Logger levellogger.Logger
37 | }
38 |
39 | func NewOptions() *Options {
40 | hostname, err := os.Hostname()
41 | if err != nil {
42 | log.Fatal(err)
43 | }
44 |
45 | return &Options{
46 | TCPAddress: "0.0.0.0:4160",
47 | HTTPAddress: "0.0.0.0:4161",
48 | BroadcastAddress: hostname,
49 | BroadcastInterface: "eth0",
50 |
51 | ClusterLeadershipAddresses: "",
52 | ClusterID: "nsq-clusterid-test-only",
53 |
54 | InactiveProducerTimeout: 60 * time.Second,
55 | NsqdPingTimeout: 15 * time.Second,
56 |
57 | LogLevel: 1,
58 | LogDir: "",
59 | Logger: &levellogger.GLogger{},
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/nsqd/in_flight_pqueue.go:
--------------------------------------------------------------------------------
1 | package nsqd
2 |
3 | type inFlightPqueue []*Message
4 |
5 | func newInFlightPqueue(capacity int) inFlightPqueue {
6 | return make(inFlightPqueue, 0, capacity)
7 | }
8 |
9 | func (pq inFlightPqueue) Swap(i, j int) {
10 | pq[i], pq[j] = pq[j], pq[i]
11 | pq[i].index = i
12 | pq[j].index = j
13 | }
14 |
15 | func (pq *inFlightPqueue) Push(x *Message) {
16 | n := len(*pq)
17 | c := cap(*pq)
18 | if n+1 > c {
19 | npq := make(inFlightPqueue, n, c*2)
20 | copy(npq, *pq)
21 | *pq = npq
22 | }
23 | *pq = (*pq)[0 : n+1]
24 | x.index = n
25 | (*pq)[n] = x
26 | pq.up(n)
27 | }
28 |
29 | func (pq *inFlightPqueue) Pop() *Message {
30 | n := len(*pq)
31 | c := cap(*pq)
32 | pq.Swap(0, n-1)
33 | pq.down(0, n-1)
34 | if n < (c/2) && c > 25 {
35 | npq := make(inFlightPqueue, n, c/2)
36 | copy(npq, *pq)
37 | *pq = npq
38 | }
39 | x := (*pq)[n-1]
40 | x.index = -1
41 | *pq = (*pq)[0 : n-1]
42 | return x
43 | }
44 |
45 | func (pq *inFlightPqueue) Remove(i int) *Message {
46 | n := len(*pq)
47 | if n-1 != i {
48 | pq.Swap(i, n-1)
49 | pq.down(i, n-1)
50 | pq.up(i)
51 | }
52 | x := (*pq)[n-1]
53 | x.index = -1
54 | *pq = (*pq)[0 : n-1]
55 | return x
56 | }
57 |
58 | func (pq *inFlightPqueue) PeekAndShift(max int64) (*Message, int64) {
59 | if len(*pq) == 0 {
60 | return nil, 0
61 | }
62 |
63 | x := (*pq)[0]
64 | if x.pri > max {
65 | return nil, x.pri - max
66 | }
67 | pq.Pop()
68 |
69 | return x, 0
70 | }
71 |
72 | func (pq *inFlightPqueue) up(j int) {
73 | for {
74 | i := (j - 1) / 2 // parent
75 | if i == j || (*pq)[j].pri >= (*pq)[i].pri {
76 | break
77 | }
78 | pq.Swap(i, j)
79 | j = i
80 | }
81 | }
82 |
83 | func (pq *inFlightPqueue) down(i, n int) {
84 | for {
85 | j1 := 2*i + 1
86 | if j1 >= n || j1 < 0 { // j1 < 0 after int overflow
87 | break
88 | }
89 | j := j1 // left child
90 | if j2 := j1 + 1; j2 < n && (*pq)[j1].pri >= (*pq)[j2].pri {
91 | j = j2 // = 2*i + 2 // right child
92 | }
93 | if (*pq)[j].pri >= (*pq)[i].pri {
94 | break
95 | }
96 | pq.Swap(i, j)
97 | i = j
98 | }
99 | }
100 |
--------------------------------------------------------------------------------
/nsqlookupd_migrate/migrate_config.go:
--------------------------------------------------------------------------------
1 | package nsqlookupd_migrate
2 |
3 | import (
4 | "strings"
5 | "errors"
6 | "fmt"
7 | "regexp"
8 | )
9 |
10 | type MigrateConfig struct {
11 | App string
12 | Key string
13 | ReqType string
14 | Env string
15 | Switches map[string]int
16 | }
17 |
18 | var cnfLog *MigrateLogger
19 | var keyFormat = "%s.to.%s"
20 | var appFormat = "nsqlookupd.migrate"
21 | var typeFormat = "COMBINATION"
22 | var IP_REGEXP = "(\\d{1,3}\\.){3}\\d{1,3}(:\\d{1,5})?"
23 | func parseClusterName(lookupdAddr string) string {
24 | match, _ := regexp.MatchString(IP_REGEXP, lookupdAddr)
25 | if match == true {
26 | re := regexp.MustCompile(IP_REGEXP)
27 | return re.FindString(lookupdAddr)
28 | } else {
29 | return strings.Split(strings.Split(lookupdAddr, ".")[0], "http://")[1]
30 | }
31 | }
32 |
33 | func initKey(originLookupd, targetLookupd, key_in_cnf string) (string, error) {
34 | var hostOri, hostTar string
35 | if key_in_cnf != "" {
36 | cnfLog.Info("key in config applied: %v", key_in_cnf)
37 | return key_in_cnf, nil
38 | }
39 | if hostOri = parseClusterName(originLookupd); hostOri == "" {
40 | msg := fmt.Sprintf("Fail to parse NSQ cluster id from %v.", originLookupd)
41 | cnfLog.Error(msg)
42 | return "", errors.New(msg)
43 | }
44 |
45 | if hostTar = parseClusterName(targetLookupd); hostTar == "" {
46 | msg := fmt.Sprintf("Fail to parse NSQ cluster id from %v.", targetLookupd)
47 | cnfLog.Error(msg)
48 | return "", errors.New(msg)
49 | }
50 |
51 | return fmt.Sprintf(keyFormat, hostOri, hostTar), nil
52 | }
53 |
54 | /**
55 | init and answer a new migrate config with no topic switches
56 | */
57 | func NewMigrateConfig(context *Context) (*MigrateConfig, error) {
58 | cnfLog = context.Logger
59 | app := appFormat
60 | key, err := initKey(context.LookupAddrOri, context.LookupAddrTar, context.Migrate_key)
61 | if err != nil {
62 | return nil, err
63 | }
64 | typeVal := typeFormat
65 | cnfLog.Info("New migrate config initialized: %v, %v, %v, %v", app, key, typeVal, context.Env)
66 | return &MigrateConfig{
67 | App: app,
68 | Key: key,
69 | ReqType: typeVal,
70 | Env: context.Env,
71 | Switches: make(map[string]int),
72 | }, nil
73 | }
74 |
75 |
--------------------------------------------------------------------------------
/nsqadmin/.eslintrc:
--------------------------------------------------------------------------------
1 | {
2 | "env": {
3 | "browser": true
4 | },
5 | // Rule docs: http://eslint.org/docs/rules/
6 | "rules": {
7 | "block-scoped-var": [1],
8 | "brace-style": [1, "1tbs", {"allowSingleLine": true}],
9 | "camelcase": [1],
10 | "comma-spacing": [1],
11 | "comma-style": [1],
12 | "computed-property-spacing": [1, "never"],
13 | "consistent-return": [1],
14 | "consistent-this": [1, "self"],
15 | "curly": [2],
16 | "dot-notation": [0],
17 | "eol-last": [1],
18 | "eqeqeq": [1],
19 | "indent": [1, 4],
20 | "key-spacing": [1],
21 | "max-len": [1, 100],
22 | "max-nested-callbacks": [2, 3], // ????
23 | "new-cap": [1],
24 | "new-parens": [1],
25 | "no-caller": [2],
26 | "no-console": [0],
27 | "no-eval": [2],
28 | "no-extend-native": [2],
29 | "no-extra-bind": [1],
30 | "no-floating-decimal": [1],
31 | "no-iterator": [1],
32 | "no-lone-blocks": [1],
33 | "no-lonely-if": [1],
34 | "no-mixed-requires": [0],
35 | "no-mixed-spaces-and-tabs": [1],
36 | "no-multi-spaces": [1],
37 | "no-multi-str": [1],
38 | "no-multiple-empty-lines": [2, {"max": 2}],
39 | "no-native-reassign": [1],
40 | "no-new": [0],
41 | "no-redeclare": [1],
42 | "no-shadow": [1],
43 | "no-spaced-func": [1],
44 | "no-throw-literal": [1],
45 | "no-trailing-spaces": [1],
46 | "no-undef": [1],
47 | "no-underscore-dangle": [0],
48 | "no-unneeded-ternary": [1],
49 | "no-unused-vars": [1],
50 | "no-use-before-define": [1, "nofunc"],
51 | "no-with": [2],
52 | "one-var": [1, "never"],
53 | "quotes": [1, "single"],
54 | "radix": [1],
55 | "semi": [1, "always"],
56 | "semi-spacing": [1],
57 | "space-after-function-name": [1, "never"],
58 | "space-after-keywords": [1, "always"],
59 | "space-before-blocks": [1, "always"],
60 | "space-before-function-paren": [1, "never"],
61 | "space-in-parens": [1, "never"],
62 | "space-infix-ops": [1],
63 | "space-return-throw-case": [1],
64 | "space-unary-ops": [1],
65 | "strict": [0],
66 | "wrap-iife": [1]
67 | },
68 | "globals": {
69 | "module": true,
70 | "require": true
71 | }
72 | }
73 |
--------------------------------------------------------------------------------
/internal/ext/ext.go:
--------------------------------------------------------------------------------
1 | package ext
2 |
3 | import (
4 | "fmt"
5 | "regexp"
6 | )
7 |
8 | const (
9 | E_EXT_NOT_SUPPORT = "E_EXT_NOT_SUPPORT"
10 | E_BAD_TAG = "E_BAD_TAG"
11 | E_INVALID_JSON_HEADER = "E_INVALID_JSON_HEADER"
12 |
13 | CLIENT_DISPATCH_TAG_KEY = "##client_dispatch_tag"
14 | TRACE_ID_KEY = "##trace_id"
15 | MaxExtLen = 65535
16 | ZAN_TEST_KEY = "zan_test"
17 | )
18 |
19 | var MAX_TAG_LEN = 100
20 | var validTagFmt = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`)
21 |
22 | type ExtVer uint8
23 |
24 | //ext versions
25 | // version for message has no ext
26 | var NO_EXT_VER = ExtVer(uint8(0))
27 |
28 | // version for message has tag ext
29 | var TAG_EXT_VER = ExtVer(uint8(2))
30 |
31 | // version for message has json header ext
32 | var JSON_HEADER_EXT_VER = ExtVer(uint8(4))
33 |
34 | var noExt = NoExt{}
35 |
36 | type NoExt []byte
37 |
38 | func NewNoExt() NoExt {
39 | return noExt
40 | }
41 |
42 | func (n NoExt) ExtVersion() ExtVer {
43 | return NO_EXT_VER
44 | }
45 |
46 | func (n NoExt) GetBytes() []byte {
47 | return nil
48 | }
49 |
50 | type TagExt []byte
51 |
52 | func NewTagExt(tagName []byte) (TagExt, error) {
53 | if err := ValidateTag(string(tagName)); err != nil {
54 | return nil, err
55 | }
56 | return TagExt(tagName), nil
57 | }
58 |
59 | func (tag TagExt) GetTagName() string {
60 | return string(tag)
61 | }
62 |
63 | func ValidateTag(beValidated string) error {
64 | lenTag := len(beValidated)
65 | if lenTag > MAX_TAG_LEN {
66 | return fmt.Errorf("invalid tag len %v, exceed limit %v", lenTag, MAX_TAG_LEN)
67 | }
68 | if !validTagFmt.Match([]byte(beValidated)) {
69 | return fmt.Errorf("invalid tag %v", beValidated)
70 | }
71 | return nil
72 | }
73 |
74 | type JsonHeaderExt struct {
75 | bytes []byte
76 | }
77 |
78 | func NewJsonHeaderExt() *JsonHeaderExt {
79 | return &JsonHeaderExt{}
80 | }
81 |
82 | func (self *JsonHeaderExt) SetJsonHeaderBytes(jsonExtBytes []byte) {
83 | self.bytes = jsonExtBytes[0:]
84 | }
85 |
86 | func (self *JsonHeaderExt) ExtVersion() ExtVer {
87 | return JSON_HEADER_EXT_VER
88 | }
89 |
90 | func (self *JsonHeaderExt) GetBytes() []byte {
91 | return self.bytes
92 | }
93 |
94 | type IExtContent interface {
95 | ExtVersion() ExtVer
96 | GetBytes() []byte
97 | }
98 |
--------------------------------------------------------------------------------
/apps/nsqlookupd_migrate_proxy/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "flag"
5 | "net/http"
6 | migrate "github.com/youzan/nsq/nsqlookupd_migrate"
7 |
8 | "log"
9 | "github.com/BurntSushi/toml"
10 | "github.com/mreiferson/go-options"
11 | "os"
12 | "github.com/absolute8511/glog"
13 | "time"
14 | )
15 |
16 | var (
17 | flagSet = flag.NewFlagSet("nsqlookup_migrate", flag.ExitOnError)
18 | port = flagSet.String("http-address", "0.0.0.0:4161", ": to listen on for HTTP clients")
19 | testPort = flagSet.String("http-address-test", "0.0.0.0:4161", ": to listen on for HTTP tester")
20 | originalLookupdHttpAddr = flagSet.String("origin-lookupd-http", "http://0.0.0.0:4161", ": original lookupd to access to fetch lookup info.")
21 | targetLookupdHttpAddr = flagSet.String("target-lookupd-http", "http://0.0.0.0:4161", ": target lookupd to access to fetch lookup info.")
22 | env = flagSet.String("env", "", "env")
23 | log_level = flagSet.Int64("log-level", 2, "log level")
24 | config = flagSet.String("config", "", "path to config file")
25 | migrate_key = flagSet.String("migrate-key", "", "key for migrate switches, it not specified, migrate proxy try parse key from origin&target lookupd addresses.")
26 | log_dir = flagSet.String("log_dir", "", "dir for log files")
27 | )
28 |
29 | func main() {
30 | glog.InitWithFlag(flagSet)
31 | flagSet.Parse(os.Args[1:])
32 | var cfg map[string]interface{}
33 | if *config != "" {
34 | _, err := toml.DecodeFile(*config, &cfg)
35 | if err != nil {
36 | log.Fatalf("ERROR: failed to load config file %s - %s", *config, err.Error())
37 | }
38 | }
39 |
40 | context := &migrate.Context{}
41 | options.Resolve(context, flagSet, cfg)
42 | log.Printf("origin lookup http address: %v", context.LookupAddrOri)
43 | log.Printf("target lookup http address: %v", context.LookupAddrTar)
44 | if context.LogDir != "" {
45 | glog.SetGLogDir(context.LogDir)
46 | log.Printf("log dir: %v", context.LogDir)
47 | }
48 | glog.StartWorker(time.Second * 2)
49 |
50 | httpServer, err := migrate.NewHTTPServer(context)
51 | if err != nil {
52 | panic(err)
53 | }
54 | log.Printf("http server listen on %v", context.ProxyHttpAddr)
55 | log.Fatal(http.ListenAndServe(context.ProxyHttpAddr, httpServer.Router))
56 | }
--------------------------------------------------------------------------------
/nsqadmin/static/js/models/topic.js:
--------------------------------------------------------------------------------
1 | var _ = require('underscore');
2 | var AppState = require('../app_state');
3 | var Backbone = require('backbone');
4 |
5 | var Topic = Backbone.Model.extend({
6 | idAttribute: 'name',
7 |
8 | constructor: function Topic() {
9 | Backbone.Model.prototype.constructor.apply(this, arguments);
10 | },
11 |
12 | url: function() {
13 | return AppState.url('/topics/' + encodeURIComponent(this.get('name')));
14 | },
15 |
16 | parse: function(response) {
17 | response['has_client_pub'] = false;
18 | response['nodes'] = _.map(response['nodes'] || [], function(node) {
19 | var nodeAddr = node['node'];
20 | var nodeParts = node['node'].split(':');
21 | var port = nodeParts.pop();
22 | var address = nodeParts.join(':');
23 | var hostname = node['hostname'];
24 | node['show_broadcast_address'] = hostname.toLowerCase() !== address.toLowerCase();
25 | node['hostname_port'] = hostname + ':' + port;
26 | node['client_pub_stats'] = _.map(node['client_pub_stats'] || [], function(pub_stats){
27 | var date = new Date(pub_stats['last_pub_ts']*1000);
28 | pub_stats['last_pub_ts'] = date.toString();
29 | return pub_stats;
30 | });
31 | if(node['client_pub_stats'].length > 0) {
32 | response['has_client_pub'] = true;
33 | }
34 | if(node['partition_hourly_pubsize']){
35 | node['partition_hourly_pubsize'].reverse();
36 | node['hourly_pubsize'] = node['partition_hourly_pubsize'][0];
37 | }
38 | return node;
39 | });
40 |
41 | var total_hourly_pubsize = new Array();
42 | _.each(response['nodes'], function(node, outIdx){
43 | _.each(node['partition_hourly_pubsize'], function(value, idx){
44 | if(total_hourly_pubsize[idx]) {
45 | total_hourly_pubsize[idx] += value;
46 | }else{
47 | total_hourly_pubsize[idx] = value;
48 | }
49 | });
50 | });
51 | response['total_partition_hourly_pubsize'] = total_hourly_pubsize;
52 | response['total_hourly_pubsize'] = response['total_partition_hourly_pubsize'][0];
53 | return response;
54 | }
55 | });
56 |
57 | module.exports = Topic;
58 |
--------------------------------------------------------------------------------
/internal/http_api/compress.go:
--------------------------------------------------------------------------------
1 | // Copyright 2013 The Gorilla Authors. All rights reserved.
2 | // Use of this source code is governed by a BSD-style
3 | // license that can be found in the LICENSE file.
4 |
5 | // copied from https://github.com/gorilla/handlers/blob/master/compress.go
6 |
7 | package http_api
8 |
9 | import (
10 | "compress/flate"
11 | "compress/gzip"
12 | "io"
13 | "net/http"
14 | "strings"
15 | )
16 |
17 | type compressResponseWriter struct {
18 | io.Writer
19 | http.ResponseWriter
20 | http.Hijacker
21 | }
22 |
23 | func (w *compressResponseWriter) Header() http.Header {
24 | return w.ResponseWriter.Header()
25 | }
26 |
27 | func (w *compressResponseWriter) WriteHeader(c int) {
28 | w.ResponseWriter.Header().Del("Content-Length")
29 | w.ResponseWriter.WriteHeader(c)
30 | }
31 |
32 | func (w *compressResponseWriter) Write(b []byte) (int, error) {
33 | h := w.ResponseWriter.Header()
34 | if h.Get("Content-Type") == "" {
35 | h.Set("Content-Type", http.DetectContentType(b))
36 | }
37 | h.Del("Content-Length")
38 | return w.Writer.Write(b)
39 | }
40 |
41 | // CompressHandler gzip compresses HTTP responses for clients that support it
42 | // via the 'Accept-Encoding' header.
43 | func CompressHandler(h http.Handler) http.Handler {
44 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
45 | L:
46 | for _, enc := range strings.Split(r.Header.Get("Accept-Encoding"), ",") {
47 | switch strings.TrimSpace(enc) {
48 | case "gzip":
49 | w.Header().Set("Content-Encoding", "gzip")
50 | w.Header().Add("Vary", "Accept-Encoding")
51 |
52 | gw := gzip.NewWriter(w)
53 | defer gw.Close()
54 |
55 | h, hok := w.(http.Hijacker)
56 | if !hok { /* w is not Hijacker... oh well... */
57 | h = nil
58 | }
59 |
60 | w = &compressResponseWriter{
61 | Writer: gw,
62 | ResponseWriter: w,
63 | Hijacker: h,
64 | }
65 |
66 | break L
67 | case "deflate":
68 | w.Header().Set("Content-Encoding", "deflate")
69 | w.Header().Add("Vary", "Accept-Encoding")
70 |
71 | fw, _ := flate.NewWriter(w, flate.DefaultCompression)
72 | defer fw.Close()
73 |
74 | h, hok := w.(http.Hijacker)
75 | if !hok { /* w is not Hijacker... oh well... */
76 | h = nil
77 | }
78 |
79 | w = &compressResponseWriter{
80 | Writer: fw,
81 | ResponseWriter: w,
82 | Hijacker: h,
83 | }
84 |
85 | break L
86 | }
87 | }
88 |
89 | h.ServeHTTP(w, r)
90 | })
91 | }
92 |
--------------------------------------------------------------------------------
/jepsen/test/nsq/core_test.clj:
--------------------------------------------------------------------------------
1 | (ns nsq.core-test
2 | (:use nsq.core
3 | jepsen.tests
4 | clojure.test
5 | clojure.pprint)
6 | (:require [clojure.string :as str]
7 | [jepsen.util :as util]
8 | [jepsen.core :as jepsen]
9 | [jepsen.os.debian :as debian]
10 | [jepsen.checker :as checker]
11 | [jepsen.checker.timeline :as timeline]
12 | [jepsen.model :as model]
13 | [jepsen.generator :as gen]
14 | [jepsen.nemesis :as nemesis]
15 | [jepsen.store :as store]
16 | [jepsen.report :as report]))
17 |
18 | (deftest nsq-test
19 | (let [test (jepsen/run!
20 | (assoc
21 | noop-test
22 | :name "nsq-simple-partition"
23 | :os debian/os
24 | :db db
25 | :client (queue-client)
26 | :nemesis (nemesis/partition-random-halves)
27 | ;:nemesis (nemesis/partition-random-node)
28 | ;:nemesis (nemesis/partition-majorities-ring)
29 | :model (model/unordered-queue)
30 | :checker (checker/compose
31 | {:total-queue checker/total-queue})
32 | :generator (gen/phases
33 | (->> (gen/queue)
34 | (gen/delay 1/10)
35 | (gen/nemesis
36 | (gen/seq
37 | (cycle [(gen/sleep 60)
38 | {:type :info :f :start}
39 | (gen/sleep 60)
40 | {:type :info :f :stop}])))
41 | (gen/time-limit 560))
42 | (gen/nemesis
43 | (gen/once {:type :info, :f :stop}))
44 | (gen/log "waiting for recovery")
45 | (gen/sleep 60)
46 | (gen/clients
47 | (gen/each
48 | (gen/once {:type :invoke
49 | :f :drain}))))))]
50 | (is (:valid? (:results test)))
51 | (report/to "report/queue.txt"
52 | (-> test :results pprint))))
53 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/youzan/nsq
2 |
3 | go 1.13
4 |
5 | require (
6 | github.com/BurntSushi/toml v0.3.1
7 | github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect
8 | github.com/Workiva/go-datastructures v1.0.50
9 | github.com/absolute8511/bolt v1.5.2
10 | github.com/absolute8511/glog v0.4.1
11 | github.com/absolute8511/gorpc v0.0.0-20161203145636-60ee7d4359cb
12 | github.com/absolute8511/goskiplist v0.0.0-20170727031420-3ba6f667c3df
13 | github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932
14 | github.com/bitly/go-simplejson v0.5.0
15 | github.com/bitly/timer_metrics v0.0.0-20170606164300-b1c65ca7ae62
16 | github.com/blang/semver v3.5.1+incompatible
17 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
18 | github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b
19 | github.com/cenkalti/backoff v2.1.0+incompatible
20 | github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
21 | github.com/cockroachdb/pebble v0.0.0-20210322142411-65860c8c27ac
22 | github.com/coreos/etcd v3.3.13+incompatible
23 | github.com/getsentry/raven-go v0.2.0 // indirect
24 | github.com/go-ole/go-ole v1.2.5 // indirect
25 | github.com/gobwas/glob v0.2.3
26 | github.com/gogo/protobuf v1.3.1
27 | github.com/golang/protobuf v1.4.2
28 | github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf
29 | github.com/gorilla/sessions v1.1.3
30 | github.com/hashicorp/golang-lru v0.5.3
31 | github.com/json-iterator/go v1.1.10
32 | github.com/judwhite/go-svc v1.0.0
33 | github.com/julienschmidt/httprouter v1.2.0
34 | github.com/kr/pretty v0.2.0 // indirect
35 | github.com/mreiferson/go-options v0.0.0-20161229190002-77551d20752b
36 | github.com/myesui/uuid v1.0.0 // indirect
37 | github.com/prometheus/client_golang v1.7.1
38 | github.com/shirou/gopsutil v3.21.2+incompatible
39 | github.com/spaolacci/murmur3 v1.1.0
40 | github.com/stretchr/testify v1.6.1
41 | github.com/tidwall/gjson v1.1.3
42 | github.com/tidwall/match v1.0.1 // indirect
43 | github.com/twinj/uuid v1.0.0
44 | github.com/twmb/murmur3 v1.1.5
45 | github.com/valyala/fastjson v1.6.1
46 | github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8
47 | github.com/youzan/go-nsq v1.7.2-HA
48 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7
49 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
50 | golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
51 | google.golang.org/grpc v1.29.1
52 | gopkg.in/stretchr/testify.v1 v1.2.2 // indirect
53 | gopkg.in/yaml.v2 v2.3.0
54 | )
55 |
56 | replace github.com/ugorji/go => github.com/ugorji/go/codec v1.1.7
57 |
--------------------------------------------------------------------------------
/internal/util/rename_windows_test.go:
--------------------------------------------------------------------------------
1 | // +build windows
2 |
3 | package util
4 |
5 | import (
6 | "fmt"
7 | "io/ioutil"
8 | "math/rand"
9 | "os"
10 | "path/filepath"
11 | "strings"
12 | "testing"
13 | "time"
14 | )
15 |
16 | const TEST_FILE_COUNT = 500
17 |
18 | func TestConcurrentRenames(t *testing.T) {
19 | var waitGroup util.WaitGroupWrapper
20 |
21 | r := rand.New(rand.NewSource(time.Now().UnixNano()))
22 | trigger := make(chan struct{})
23 | testDir := filepath.Join(os.TempDir(), fmt.Sprintf("nsqd_TestConcurrentRenames_%d", r.Int()))
24 |
25 | err := os.MkdirAll(testDir, 644)
26 | if err != nil {
27 | t.Error(err)
28 | }
29 |
30 | fis, err := ioutil.ReadDir(testDir)
31 | if err != nil {
32 | t.Error(err)
33 | } else if len(fis) > 0 {
34 | t.Errorf("Test directory %s unexpectedly has %d items in it!", testDir, len(fis))
35 | t.FailNow()
36 | }
37 |
38 | // create a bunch of source files and attempt to concurrently rename them all
39 | for i := 1; i <= TEST_FILE_COUNT; i++ {
40 | //First rename doesn't overwrite/replace; no target present
41 | sourcePath1 := filepath.Join(testDir, fmt.Sprintf("source1_%d.txt", i))
42 | //Second rename will replace
43 | sourcePath2 := filepath.Join(testDir, fmt.Sprintf("source2_%d.txt", i))
44 | targetPath := filepath.Join(testDir, fmt.Sprintf("target_%d.txt", i))
45 | err = ioutil.WriteFile(sourcePath1, []byte(sourcePath1), 0644)
46 | if err != nil {
47 | t.Error(err)
48 | }
49 | err = ioutil.WriteFile(sourcePath2, []byte(sourcePath2), 0644)
50 | if err != nil {
51 | t.Error(err)
52 | }
53 |
54 | waitGroup.Wrap(func() {
55 | _, _ = <-trigger
56 | err := AtomicRename(sourcePath1, targetPath)
57 | if err != nil {
58 | t.Error(err)
59 | }
60 | err = AtomicRename(sourcePath2, targetPath)
61 | if err != nil {
62 | t.Error(err)
63 | }
64 | })
65 | }
66 |
67 | // start.. they're off to the races!
68 | close(trigger)
69 |
70 | // wait for completion...
71 | waitGroup.Wait()
72 |
73 | // no source files should exist any longer; we should just have 500 target files
74 | fis, err = ioutil.ReadDir(testDir)
75 | if err != nil {
76 | t.Error(err)
77 | } else if len(fis) != TEST_FILE_COUNT {
78 | t.Errorf("Test directory %s unexpectedly has %d items in it!", testDir, len(fis))
79 | } else {
80 | for _, fi := range fis {
81 | if !strings.HasPrefix(fi.Name(), "target_") {
82 | t.Errorf("Test directory file %s is not expected target file!", fi.Name())
83 | }
84 | }
85 | }
86 |
87 | // clean up the test directory
88 | err = os.RemoveAll(testDir)
89 | if err != nil {
90 | t.Error(err)
91 | }
92 | }
93 |
--------------------------------------------------------------------------------
/internal/quantile/aggregate.go:
--------------------------------------------------------------------------------
1 | package quantile
2 |
3 | import (
4 | "encoding/json"
5 | "math"
6 | "sort"
7 | )
8 |
9 | type E2eProcessingLatencyAggregate struct {
10 | Count int `json:"count"`
11 | Percentiles []map[string]float64 `json:"percentiles"`
12 | Topic string `json:"topic"`
13 | Channel string `json:"channel"`
14 | Addr string `json:"host"`
15 | }
16 |
17 | func (e *E2eProcessingLatencyAggregate) UnmarshalJSON(b []byte) error {
18 | var resp struct {
19 | Count int `json:"count"`
20 | Percentiles []map[string]float64 `json:"percentiles"`
21 | Topic string `json:"topic"`
22 | Channel string `json:"channel"`
23 | Addr string `json:"host"`
24 | }
25 | err := json.Unmarshal(b, &resp)
26 | if err != nil {
27 | return err
28 | }
29 |
30 | for _, p := range resp.Percentiles {
31 | p["min"] = p["value"]
32 | p["max"] = p["value"]
33 | p["average"] = p["value"]
34 | p["count"] = float64(resp.Count)
35 | }
36 |
37 | e.Count = resp.Count
38 | e.Percentiles = resp.Percentiles
39 | e.Topic = resp.Topic
40 | e.Channel = resp.Channel
41 | e.Addr = resp.Addr
42 |
43 | return nil
44 | }
45 |
46 | func (e *E2eProcessingLatencyAggregate) Len() int { return len(e.Percentiles) }
47 | func (e *E2eProcessingLatencyAggregate) Swap(i, j int) {
48 | e.Percentiles[i], e.Percentiles[j] = e.Percentiles[j], e.Percentiles[i]
49 | }
50 | func (e *E2eProcessingLatencyAggregate) Less(i, j int) bool {
51 | return e.Percentiles[i]["percentile"] > e.Percentiles[j]["percentile"]
52 | }
53 |
54 | // Add merges e2 into e by averaging the percentiles
55 | func (e *E2eProcessingLatencyAggregate) Add(e2 *E2eProcessingLatencyAggregate) {
56 | e.Addr = "*"
57 | p := e.Percentiles
58 | e.Count += e2.Count
59 | for _, value := range e2.Percentiles {
60 | i := -1
61 | for j, v := range p {
62 | if value["quantile"] == v["quantile"] {
63 | i = j
64 | break
65 | }
66 | }
67 | if i == -1 {
68 | i = len(p)
69 | e.Percentiles = append(p, make(map[string]float64))
70 | p = e.Percentiles
71 | p[i]["quantile"] = value["quantile"]
72 | }
73 | p[i]["max"] = math.Max(value["max"], p[i]["max"])
74 | p[i]["min"] = math.Min(value["max"], p[i]["max"])
75 | p[i]["count"] += value["count"]
76 | if p[i]["count"] == 0 {
77 | p[i]["average"] = 0
78 | continue
79 | }
80 | delta := value["average"] - p[i]["average"]
81 | R := delta * value["count"] / p[i]["count"]
82 | p[i]["average"] = p[i]["average"] + R
83 | }
84 | sort.Sort(e)
85 | }
86 |
--------------------------------------------------------------------------------
/nsqadmin/static/js/views/nodes.hbs:
--------------------------------------------------------------------------------
1 | {{> warning}}
2 | {{> error}}
3 |
4 |
5 |
6 |
Cluster Status
7 |
8 | {{#each clusters}}
9 |
{{dc}} {{#if stable}}Stable {{else}}Not Stable {{/if}}
10 | {{/each}}
11 |
12 |
13 |
16 |
17 |
NSQd Nodes ({{collection.length}})
18 |
19 |
22 |
23 |
24 |
25 | {{#if dcnsqlookupd.length}}
26 | DC
27 | {{/if}}
28 | Hostname
29 | Broadcast Address
30 | TCP Port
31 | HTTP Port
32 | Version
33 | {{#if nsqlookupd.length}}
34 | Lookupd Conns.
35 | {{/if}}
36 | Topics
37 |
38 | {{#each collection}}
39 |
40 | {{#if ../dcnsqlookupd.length}}
41 | {{dc}}
42 | {{/if}}
43 | {{hostname}}
44 | {{broadcast_address}} {{tcp_port}}
46 | {{http_port}}
47 | {{version}}
48 | {{#if ../nsqlookupd.length}}
49 |
50 | {{remote_addresses.length}}
51 |
52 | {{#each remote_addresses}}{{this}}
54 |
55 | {{/if}}
56 |
57 | {{#if topics.length}}
58 | {{topics.length}}
59 | {{#each topics}}
60 | {{topic}}
61 | {{/each}}
62 | {{/if}}
63 |
64 |
65 | {{/each}}
66 |
67 |
68 |
: to connect to nsqd")
17 | )
18 |
19 | func main() {
20 | flag.Parse()
21 | var wg sync.WaitGroup
22 |
23 | goChan := make(chan int)
24 | rdyChan := make(chan int)
25 | conn, err := net.DialTimeout("tcp", *tcpAddress, time.Second)
26 | if err != nil {
27 | panic(err.Error())
28 | }
29 | conn.Write(nsq.MagicV2)
30 |
31 | for j := 0; j < *num; j++ {
32 | nsq.CreateTopic(fmt.Sprintf("t%d", j), 0).WriteTo(conn)
33 | resp, err := nsq.ReadResponse(conn)
34 | if err != nil {
35 | panic(err.Error())
36 | }
37 | frameType, data, err := nsq.UnpackResponse(resp)
38 | if err != nil {
39 | panic(err.Error())
40 | }
41 | if frameType == nsq.FrameTypeError {
42 | panic(string(data))
43 | }
44 |
45 | }
46 |
47 | for j := 0; j < *num; j++ {
48 | wg.Add(1)
49 | go func(id int) {
50 | subWorker(*num, *tcpAddress, fmt.Sprintf("t%d", j), "ch", rdyChan, goChan, id)
51 | wg.Done()
52 | }(j)
53 | <-rdyChan
54 | time.Sleep(5 * time.Millisecond)
55 | }
56 |
57 | close(goChan)
58 | wg.Wait()
59 | }
60 |
61 | func subWorker(n int, tcpAddr string,
62 | topic string, channel string,
63 | rdyChan chan int, goChan chan int, id int) {
64 | conn, err := net.DialTimeout("tcp", tcpAddr, time.Second)
65 | if err != nil {
66 | panic(err.Error())
67 | }
68 | conn.Write(nsq.MagicV2)
69 | rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
70 | ci := make(map[string]interface{})
71 | ci["short_id"] = "test"
72 | ci["long_id"] = "test"
73 | cmd, _ := nsq.Identify(ci)
74 | cmd.WriteTo(rw)
75 | nsq.Subscribe(topic, channel).WriteTo(rw)
76 | rdyCount := 1
77 | rdy := rdyCount
78 | rdyChan <- 1
79 | <-goChan
80 | nsq.Ready(rdyCount).WriteTo(rw)
81 | rw.Flush()
82 | nsq.ReadResponse(rw)
83 | nsq.ReadResponse(rw)
84 | for {
85 | resp, err := nsq.ReadResponse(rw)
86 | if err != nil {
87 | panic(err.Error())
88 | }
89 | frameType, data, err := nsq.UnpackResponse(resp)
90 | if err != nil {
91 | panic(err.Error())
92 | }
93 | if frameType == nsq.FrameTypeError {
94 | panic(string(data))
95 | } else if frameType == nsq.FrameTypeResponse {
96 | nsq.Nop().WriteTo(rw)
97 | rw.Flush()
98 | continue
99 | }
100 | msg, err := nsq.DecodeMessage(data)
101 | if err != nil {
102 | panic(err.Error())
103 | }
104 | nsq.Finish(msg.ID).WriteTo(rw)
105 | rdy--
106 | if rdy == 0 {
107 | nsq.Ready(rdyCount).WriteTo(rw)
108 | rdy = rdyCount
109 | rw.Flush()
110 | }
111 | }
112 | }
113 |
--------------------------------------------------------------------------------
/nsqadmin/static/js/models/channel.js:
--------------------------------------------------------------------------------
1 | var _ = require('underscore');
2 |
3 | var AppState = require('../app_state');
4 | var Backbone = require('backbone');
5 |
6 | var Channel = Backbone.Model.extend({
7 | idAttribute: 'name',
8 |
9 | constructor: function Channel() {
10 | Backbone.Model.prototype.constructor.apply(this, arguments);
11 | },
12 |
13 | url: function() {
14 | return AppState.url('/topics/' +
15 | encodeURIComponent(this.get('topic')) + '/' +
16 | encodeURIComponent(this.get('name')));
17 | },
18 |
19 | clientUrl: function() {
20 | return AppState.url('/topics/' +
21 | encodeURIComponent(this.get('topic')) + '/' +
22 | encodeURIComponent(this.get('name'))) + '/client';
23 | },
24 |
25 | parse: function(response) {
26 | response['nodes'] = _.map(response['nodes'] || [], function(node) {
27 | var nodeParts = node['node'].split(':');
28 | var port = nodeParts.pop();
29 | var address = nodeParts.join(':');
30 | var hostname = node['hostname'];
31 | node['show_broadcast_address'] = hostname.toLowerCase() !== address.toLowerCase();
32 | node['hostname_port'] = hostname + ':' + port;
33 | //parse node to limit the length
34 | if(!node['msg_consume_latency_stats']) {
35 | node['msg_consume_latency_stats'] = new Array();
36 | for(i = 0; i < 12; i++) {
37 | node['msg_consume_latency_stats'].push("n/a");
38 | }
39 | } else {
40 | var diffInLen = 12 - node['msg_consume_latency_stats'].length;
41 | if(diffInLen < 0){
42 | node['msg_consume_latency_stats'] = node['msg_consume_latency_stats'].slice(0, 12);
43 | } else {
44 | for (j = 0; j < diffInLen; j++) {
45 | node['msg_consume_latency_stats'].push("n/a");
46 | }
47 | }
48 | }
49 | return node;
50 | });
51 |
52 | response['clients'] = _.map(response['clients'] || [], function(client) {
53 | var clientId = client['client_id'];
54 | var hostname = client['hostname'];
55 | var shortHostname = hostname.split('.')[0];
56 |
57 | // ignore client_id if it's duplicative
58 | client['show_client_id'] = (clientId.toLowerCase() !== shortHostname.toLowerCase()
59 | && clientId.toLowerCase() !== hostname.toLowerCase());
60 |
61 | var port = client['remote_address'].split(':').pop();
62 | client['hostname_port'] = hostname + ':' + port;
63 |
64 | return client;
65 | });
66 |
67 | return response;
68 | }
69 | });
70 |
71 | module.exports = Channel;
72 |
--------------------------------------------------------------------------------
/consistence/coordgrpc/coord_grpc.proto:
--------------------------------------------------------------------------------
1 | syntax = "proto3";
2 | package coordgrpc;
3 |
4 | import "gogoproto/gogo.proto";
5 |
6 | option (gogoproto.marshaler_all) = true;
7 | option (gogoproto.sizer_all) = true;
8 | option (gogoproto.unmarshaler_all) = true;
9 | option (gogoproto.goproto_getters_all) = false;
10 | option (gogoproto.goproto_enum_prefix_all) = false;
11 |
12 | service NsqdCoordRpcV2 {
13 | rpc UpdateChannelOffset(RpcChannelOffsetArg) returns (CoordErr) {}
14 | rpc PutMessage(RpcPutMessage) returns (CoordErr) {}
15 | rpc PutMessages(RpcPutMessages) returns (CoordErr) {}
16 | rpc PullCommitLogsAndData(PullCommitLogsReq) returns (PullCommitLogsRsp) {}
17 | rpc PullDelayedQueueCommitLogsAndData(PullCommitLogsReq) returns (PullCommitLogsRsp) {}
18 | }
19 |
20 | message CoordErr {
21 | string err_msg = 1;
22 | int32 err_code = 2;
23 | int32 err_type = 3;
24 | }
25 |
26 | message RpcTopicData {
27 | string topic_name = 1;
28 | int32 topic_partition = 2;
29 | int64 epoch = 3;
30 | int64 topic_write_epoch = 4;
31 | int64 topic_leader_session_epoch = 5;
32 | string topic_leader_session = 6;
33 | string topic_leader = 7;
34 | }
35 |
36 | message MsgQueueInterval {
37 | int64 start = 1;
38 | int64 end = 2;
39 | uint64 end_cnt = 3;
40 | }
41 |
42 | message ChannelConsumerOffset {
43 | int64 voffset = 1;
44 | bool flush = 2;
45 | bool allow_backward = 3;
46 | int64 vcnt = 4;
47 | bool need_update_confirmed = 5;
48 | repeated MsgQueueInterval confirmed_intervals = 6 [(gogoproto.nullable) = false];
49 | }
50 |
51 | message CommitLogData {
52 | int64 logID = 1;
53 | int64 epoch = 2;
54 | int64 last_msg_logID = 3;
55 | int64 msg_offset = 4;
56 | int32 msg_size = 5;
57 | int64 msg_cnt = 6;
58 | int32 msg_num = 7;
59 | }
60 |
61 | message NsqdMessage {
62 | uint64 ID = 1;
63 | uint64 trace_ID = 2;
64 | bytes body = 3;
65 | int64 timestamp = 4;
66 | uint32 attemps = 5;
67 | }
68 |
69 | message RpcChannelOffsetArg {
70 | RpcTopicData topic_data = 1;
71 | string channel = 2;
72 | ChannelConsumerOffset channel_offset = 3;
73 | }
74 |
75 | message RpcPutMessage {
76 | RpcTopicData topic_data = 1;
77 | CommitLogData log_data = 2;
78 | NsqdMessage topic_message = 3;
79 | bytes topic_raw_message = 4;
80 | }
81 |
82 | message RpcPutMessages {
83 | RpcTopicData topic_data = 1;
84 | CommitLogData log_data = 2;
85 | repeated NsqdMessage topic_message = 3;
86 | bytes topic_raw_message = 4;
87 | }
88 |
89 | message PullCommitLogsReq {
90 | RpcTopicData topic_data = 1;
91 | int64 start_log_offset = 2;
92 | int32 log_max_num = 3;
93 | int64 start_index_cnt = 4;
94 | int64 log_count_num_index = 5;
95 | bool use_count_index = 6;
96 | }
97 |
98 | message PullCommitLogsRsp {
99 | repeated CommitLogData logs = 1 [(gogoproto.nullable) = false];
100 | repeated bytes data_list = 2;
101 | }
--------------------------------------------------------------------------------
/internal/flume_log/flume_client.go:
--------------------------------------------------------------------------------
1 | package flume_log
2 |
3 | import (
4 | "bufio"
5 | "errors"
6 | "log"
7 | "net"
8 | "time"
9 | )
10 |
11 | type FlumeClient struct {
12 | remoteAddr string
13 | conn net.Conn
14 | bw *bufio.Writer
15 | stopChan chan bool
16 | bufList chan []byte
17 | loopDone chan bool
18 | }
19 |
20 | func NewFlumeClient(agentIP string) *FlumeClient {
21 | client := &FlumeClient{
22 | bufList: make(chan []byte, 1000),
23 | stopChan: make(chan bool),
24 | remoteAddr: agentIP,
25 | loopDone: make(chan bool),
26 | }
27 |
28 | go client.writeLoop()
29 | return client
30 | }
31 |
32 | func readLoop(conn net.Conn) {
33 | buf := make([]byte, 1024)
34 | for {
35 | _, err := conn.Read(buf)
36 | if err != nil {
37 | conn.Close()
38 | return
39 | }
40 | }
41 | }
42 |
43 | func (c *FlumeClient) writeLoop() {
44 | c.reconnect()
45 | ticker := time.NewTicker(time.Second * 3)
46 | defer func() {
47 | ticker.Stop()
48 | select {
49 | case buf := <-c.bufList:
50 | _, err := c.bw.Write(buf)
51 | if err != nil {
52 | log.Printf("write log %v failed: %v, left data: %v", string(buf), err, len(c.bufList))
53 | break
54 | }
55 | default:
56 | }
57 | if c.conn != nil {
58 | c.bw.Flush()
59 | c.conn.Close()
60 | c.conn = nil
61 | }
62 | close(c.loopDone)
63 | }()
64 | for {
65 | if c.conn == nil {
66 | err := c.reconnect()
67 | if err != nil {
68 | select {
69 | case <-time.After(time.Second):
70 | case <-c.stopChan:
71 | return
72 | }
73 | continue
74 | }
75 | }
76 | select {
77 | case buf := <-c.bufList:
78 | _, err := c.bw.Write(buf)
79 | if err != nil {
80 | log.Printf("write log %v failed: %v", string(buf), err)
81 | c.reconnect()
82 | }
83 | case <-ticker.C:
84 | err := c.bw.Flush()
85 | if err != nil {
86 | log.Printf("flush write log failed: %v", err)
87 | c.reconnect()
88 | }
89 | case <-c.stopChan:
90 | return
91 | }
92 | }
93 | }
94 |
95 | func (c *FlumeClient) SendLog(d []byte) error {
96 | select {
97 | case <-c.stopChan:
98 | return errors.New("flume client stopped")
99 | case c.bufList <- d:
100 | default:
101 | return errors.New("flume client buffer overflowed")
102 | }
103 | return nil
104 | }
105 |
106 | func (c *FlumeClient) reconnect() (err error) {
107 | if c.conn != nil {
108 | c.bw.Flush()
109 | c.conn.Close()
110 | c.conn = nil
111 | c.bw = nil
112 | }
113 | log.Printf("reconnect flumelogger to %v ", c.remoteAddr)
114 | conn, err := net.DialTimeout("tcp", c.remoteAddr, time.Second*5)
115 | if err != nil {
116 | log.Printf("connect to %v failed: %v", c.remoteAddr, err)
117 | return err
118 | } else {
119 | c.conn = conn
120 | c.bw = bufio.NewWriterSize(conn, 1024*8)
121 | go readLoop(conn)
122 | }
123 | return nil
124 | }
125 |
126 | func (c *FlumeClient) Stop() {
127 | if c.stopChan != nil {
128 | close(c.stopChan)
129 | }
130 | <-c.loopDone
131 | }
132 |
--------------------------------------------------------------------------------
/nsqadmin/options.go:
--------------------------------------------------------------------------------
1 | package nsqadmin
2 |
3 | import (
4 | "time"
5 |
6 | "github.com/youzan/nsq/internal/levellogger"
7 | "github.com/youzan/nsq/internal/clusterinfo"
8 | )
9 |
10 | type Options struct {
11 | HTTPAddress string `flag:"http-address"`
12 |
13 | GraphiteURL string `flag:"graphite-url"`
14 | ProxyGraphite bool `flag:"proxy-graphite"`
15 |
16 | UseStatsdPrefixes bool `flag:"use-statsd-prefixes"`
17 | StatsdPrefix string `flag:"statsd-prefix"`
18 | StatsdCounterFormat string `flag:"statsd-counter-format"`
19 | StatsdGaugeFormat string `flag:"statsd-gauge-format"`
20 |
21 | StatsdInterval time.Duration `flag:"statsd-interval"`
22 |
23 | NSQLookupdHTTPAddresses []string `flag:"lookupd-http-address" cfg:"nsqlookupd_http_addresses"`
24 | NSQDHTTPAddresses []string `flag:"nsqd-http-address" cfg:"nsqd_http_addresses"`
25 |
26 | DCNSQLookupdHTTPAddresses []string `flag:"dc-lookupd-http-address" cfg:"dc_lookupd_http_addresses"`
27 |
28 | NSQLookupdHTTPAddressesDC []clusterinfo.LookupdAddressDC
29 |
30 | HTTPClientTLSInsecureSkipVerify bool `flag:"http-client-tls-insecure-skip-verify"`
31 | HTTPClientTLSRootCAFile string `flag:"http-client-tls-root-ca-file"`
32 | HTTPClientTLSCert string `flag:"http-client-tls-cert"`
33 | HTTPClientTLSKey string `flag:"http-client-tls-key"`
34 |
35 | NotificationHTTPEndpoint string `flag:"notification-http-endpoint"`
36 | TraceQueryURL string `flag:"trace-query-url"`
37 | TraceAppID string `flag:"trace-app-id"`
38 | TraceAppName string `flag:"trace-app-name"`
39 | TraceLogIndexID string `flag:"trace-log-index-id"`
40 | TraceLogIndexName string `flag:"trace-log-index-name"`
41 | TraceLogPageCount int `flag:"trace-log-page-count"`
42 |
43 | ChannelCreationRetry int `flag:"channel-create-retry"`
44 | ChannelCreationBackoffInterval int `flag:"channel-create-backoff-interval"`
45 |
46 | AuthUrl string `flag:"auth-url" cfg:"auth_url"`
47 | AuthSecret string `flag:"auth-secret" cfg:"auth_secret"`
48 | LogoutUrl string `flag:"logout-url" cfg:"logout_url"`
49 | AppName string `flag:"app-name" cfg:"app_name"`
50 | RedirectUrl string `flag:"redirect-url" cfg:"redirect_url"`
51 | LogDir string `flag:"log-dir" cfg:"log_dir"`
52 | Logger levellogger.Logger
53 | AccessTokens []string `flag:"access-tokens" cfg:"access_tokens"`
54 |
55 | AccessControlFile string `flag:"access-control-file"`
56 | EnableZanTestSkip bool `flag:"enable-zan-test-skip"`
57 | }
58 |
59 | func NewOptions() *Options {
60 | return &Options{
61 | HTTPAddress: "0.0.0.0:4171",
62 | UseStatsdPrefixes: true,
63 | StatsdPrefix: "nsq.%s",
64 | StatsdCounterFormat: "stats.counters.%s.count",
65 | StatsdGaugeFormat: "stats.gauges.%s",
66 | StatsdInterval: 60 * time.Second,
67 | ChannelCreationRetry: 3,
68 | ChannelCreationBackoffInterval: 1000,
69 | Logger: &levellogger.GLogger{},
70 | TraceLogPageCount: 60,
71 | }
72 | }
73 |
--------------------------------------------------------------------------------
/internal/quantile/quantile.go:
--------------------------------------------------------------------------------
1 | package quantile
2 |
3 | import (
4 | "strings"
5 | "sync"
6 | "time"
7 |
8 | "github.com/bmizerany/perks/quantile"
9 | "github.com/youzan/nsq/internal/stringy"
10 | )
11 |
12 | type Result struct {
13 | Count int `json:"count"`
14 | Percentiles []map[string]float64 `json:"percentiles"`
15 | }
16 |
17 | func (r *Result) String() string {
18 | var s []string
19 | for _, item := range r.Percentiles {
20 | s = append(s, stringy.NanoSecondToHuman(item["value"]))
21 | }
22 | return strings.Join(s, ", ")
23 | }
24 |
25 | type Quantile struct {
26 | sync.Mutex
27 | streams [2]quantile.Stream
28 | currentIndex uint8
29 | lastMoveWindow time.Time
30 | currentStream *quantile.Stream
31 |
32 | Percentiles []float64
33 | MoveWindowTime time.Duration
34 | }
35 |
36 | func New(WindowTime time.Duration, Percentiles []float64) *Quantile {
37 | q := Quantile{
38 | currentIndex: 0,
39 | lastMoveWindow: time.Now(),
40 | MoveWindowTime: WindowTime / 2,
41 | Percentiles: Percentiles,
42 | }
43 | for i := 0; i < 2; i++ {
44 | q.streams[i] = *quantile.NewTargeted(Percentiles...)
45 | }
46 | q.currentStream = &q.streams[0]
47 | return &q
48 | }
49 |
50 | func (q *Quantile) Result() *Result {
51 | if q == nil {
52 | return &Result{}
53 | }
54 | queryHandler := q.QueryHandler()
55 | result := Result{
56 | Count: queryHandler.Count(),
57 | Percentiles: make([]map[string]float64, len(q.Percentiles)),
58 | }
59 | for i, p := range q.Percentiles {
60 | value := queryHandler.Query(p)
61 | result.Percentiles[i] = map[string]float64{"quantile": p, "value": value}
62 | }
63 | return &result
64 | }
65 |
66 | func (q *Quantile) Insert(msgStartTime int64) {
67 | q.Lock()
68 |
69 | now := time.Now()
70 | for q.IsDataStale(now) {
71 | q.moveWindow()
72 | }
73 |
74 | q.currentStream.Insert(float64(now.UnixNano() - msgStartTime))
75 | q.Unlock()
76 | }
77 |
78 | func (q *Quantile) QueryHandler() *quantile.Stream {
79 | q.Lock()
80 | now := time.Now()
81 | for q.IsDataStale(now) {
82 | q.moveWindow()
83 | }
84 |
85 | merged := quantile.NewTargeted(q.Percentiles...)
86 | merged.Merge(q.streams[0].Samples())
87 | merged.Merge(q.streams[1].Samples())
88 | q.Unlock()
89 | return merged
90 | }
91 |
92 | func (q *Quantile) IsDataStale(now time.Time) bool {
93 | return now.After(q.lastMoveWindow.Add(q.MoveWindowTime))
94 | }
95 |
96 | func (q *Quantile) Merge(them *Quantile) {
97 | q.Lock()
98 | them.Lock()
99 | iUs := q.currentIndex
100 | iThem := them.currentIndex
101 |
102 | q.streams[iUs].Merge(them.streams[iThem].Samples())
103 |
104 | iUs ^= 0x1
105 | iThem ^= 0x1
106 | q.streams[iUs].Merge(them.streams[iThem].Samples())
107 |
108 | if q.lastMoveWindow.Before(them.lastMoveWindow) {
109 | q.lastMoveWindow = them.lastMoveWindow
110 | }
111 | q.Unlock()
112 | them.Unlock()
113 | }
114 |
115 | func (q *Quantile) moveWindow() {
116 | q.currentIndex ^= 0x1
117 | q.currentStream = &q.streams[q.currentIndex]
118 | q.lastMoveWindow = q.lastMoveWindow.Add(q.MoveWindowTime)
119 | q.currentStream.Reset()
120 | }
121 |
--------------------------------------------------------------------------------
/nsqadmin/static/js/views/topicsFilter.js:
--------------------------------------------------------------------------------
1 | var _ = require('underscore');
2 | var Pubsub = require('../lib/pubsub');
3 | var AppState = require('../app_state');
4 |
5 | var BaseView = require('./base');
6 | var TopicsView = require('./topics');
7 | var TopicsMeta = require('../collections/topics_meta');
8 | var metaFetched = false;
9 | var TopicsFilterView = BaseView.extend({
10 | className: 'topicsFilter container-fluid',
11 |
12 | template: require('./spinner.hbs'),
13 |
14 | events: {
15 | 'click .filters input': 'topicsFilterAction',
16 | },
17 |
18 | filterTopics: function(e) {
19 | $('.table tr:not(.title)').show();
20 |
21 | $(e.target).parent().find('input')
22 | .filter(function(){
23 | if($(this).is(':checked')) {
24 | return this;
25 | }
26 | })
27 | .each(function() {
28 | var filterValue = $(this).val();
29 | $('.table tr:not(.title)')
30 | .filter(
31 | function(){
32 | if ($(this)
33 | .find('td a#' + filterValue).length == 0){
34 | return this;
35 | }
36 | }
37 | )
38 | .hide();
39 | }
40 | );
41 | },
42 |
43 | topicsFilterAction: function(e) {
44 | e.stopPropagation();
45 | if(!metaFetched) {
46 | var topicsView = this.topicsView;
47 | var filter = this.filterTopics;
48 | var topicsMeta = new TopicsMeta();
49 | topicsMeta.fetch()
50 | .done(function(data){
51 | var topics = _.map(data['topics'], function(topic) {
52 | return {
53 | 'name': topic['topic_name'],
54 | 'extend_support': topic['extend_support'],
55 | 'ordered': topic['ordered']
56 | };
57 | });
58 |
59 | topicsView.render(
60 | {
61 | 'message': data['message'],
62 | 'collection': topics
63 | });
64 | metaFetched = true;
65 | console.log("topic meta fetched.")
66 | filter(e);
67 | });
68 | } else {
69 | this.filterTopics(e);
70 | }
71 |
72 | },
73 |
74 | initialize: function(options){
75 | metaFetched = false;
76 | this.template = require('./topicsFilter.hbs');
77 | this.topicsView = new TopicsView();
78 | },
79 |
80 | render: function() {
81 | BaseView.prototype.initialize.apply(this, arguments);
82 | this.$el.html(this.template());
83 | this.$el.prepend(this.topicsView.render().el);
84 | return this;
85 | }
86 | });
87 |
88 | module.exports = TopicsFilterView;
--------------------------------------------------------------------------------
/nsqadmin/static/js/views/header.hbs:
--------------------------------------------------------------------------------
1 |
2 |
3 |
12 |
13 |
41 |
42 | {{#if auth_enabled}}
43 | {{#if login}}
44 | 欢迎 {{user}} Log out Sign in v{{version}}
52 |
53 |
55 |
--------------------------------------------------------------------------------
/nsqadmin/whitelist.go:
--------------------------------------------------------------------------------
1 | package nsqadmin
2 |
3 | import (
4 | "io/ioutil"
5 | "os"
6 | "sync"
7 | "time"
8 |
9 | "gopkg.in/yaml.v2"
10 | )
11 |
12 | type AccessControl interface {
13 | IsAdmin(username string) bool
14 | Start()
15 | Stop()
16 | }
17 |
18 | type YamlAccessControl struct {
19 | filePath string
20 | accessMap map[string]interface{}
21 | lock sync.RWMutex
22 | updateTicker *time.Ticker
23 | tStopChan chan int
24 | wg sync.WaitGroup
25 | lastUpdated time.Time
26 | ctx *Context
27 | }
28 |
29 | func readYml(path string) (map[string]interface{}, error) {
30 | buf, err := ioutil.ReadFile(path)
31 | if err != nil {
32 | return nil, err
33 | }
34 |
35 | m := make(map[string]interface{})
36 | err = yaml.Unmarshal(buf, &m)
37 | if err != nil {
38 | return nil, err
39 | }
40 | return m, nil
41 | }
42 |
43 | func NewYamlAccessControl(ctx *Context, filePath string) (AccessControl, error) {
44 | if filePath == "" {
45 | return nil, nil
46 | }
47 | acMp, err := readYml(filePath)
48 | if err != nil {
49 | return nil, err
50 | }
51 | info, err := os.Stat(filePath)
52 | if err != nil {
53 | return nil, err
54 | }
55 | mt := info.ModTime()
56 |
57 | return &YamlAccessControl{
58 | filePath: filePath,
59 | accessMap: acMp,
60 | lastUpdated: mt,
61 | ctx: ctx,
62 | }, nil
63 | }
64 |
65 | func (ac *YamlAccessControl) Start() {
66 | ac.tStopChan = make(chan int, 1)
67 | ac.updateTicker = time.NewTicker(10 * time.Second)
68 | ac.wg.Add(1)
69 | go func() {
70 | defer ac.wg.Done()
71 | defer ac.updateTicker.Stop()
72 | for {
73 | select {
74 | case <-ac.updateTicker.C:
75 | {
76 | //update access control map
77 | ac.tryUpdateControlFile()
78 | }
79 | case <-ac.tStopChan:
80 | {
81 | return
82 | }
83 | }
84 | }
85 | }()
86 | }
87 |
88 | func (ac *YamlAccessControl) Stop() {
89 | close(ac.tStopChan)
90 | ac.wg.Wait()
91 | }
92 |
93 | func (ac *YamlAccessControl) IsAdmin(username string) bool {
94 | ac.lock.RLock()
95 | defer ac.lock.RUnlock()
96 | //anyone is admin if access control file is disabled
97 | if !ac.isEnable() {
98 | return true
99 | }
100 | if admins, exist := ac.accessMap["admin"]; !exist {
101 | return false
102 | } else {
103 | //ac.ctx.nsqadmin.logf("admins map %v", admins)
104 | if isAdmin, exist := admins.(map[string]interface{})[username]; !exist {
105 | return false
106 | } else {
107 | return isAdmin.(bool)
108 | }
109 | }
110 | }
111 |
112 | //read action in access control file DO NOT add read lock
113 | func (ac *YamlAccessControl) isEnable() bool {
114 | if enable, exist := ac.accessMap["enable"]; !exist {
115 | return false
116 | } else {
117 | return enable.(bool)
118 | }
119 | }
120 |
121 | func (ac *YamlAccessControl) tryUpdateControlFile() {
122 | info, err := os.Stat(ac.filePath)
123 | if err != nil {
124 | ac.ctx.nsqadmin.logf("ERROR: fail to fetch info for access control file: %v", ac.filePath)
125 | return
126 | }
127 | mt := info.ModTime()
128 | //compare with last modified time
129 | if mt.After(ac.lastUpdated) {
130 | ac.ctx.nsqadmin.logf("INFO: access control file modified, try updating")
131 | newAcMap, err := readYml(ac.filePath)
132 | if err != nil {
133 | ac.ctx.nsqadmin.logf("ERROR: fail to read access control file from %v", ac.filePath)
134 | return
135 | }
136 | ac.lock.Lock()
137 | defer ac.lock.Unlock()
138 | ac.accessMap = newAcMap
139 | ac.lastUpdated = mt
140 | }
141 | }
142 |
--------------------------------------------------------------------------------
/internal/auth/authorizations.go:
--------------------------------------------------------------------------------
1 | package auth
2 |
3 | import (
4 | "errors"
5 | "fmt"
6 | "log"
7 | "net/url"
8 | "regexp"
9 | "time"
10 |
11 | "github.com/youzan/nsq/internal/http_api"
12 | )
13 |
14 | type Authorization struct {
15 | Topic string `json:"topic"`
16 | Channels []string `json:"channels"`
17 | Permissions []string `json:"permissions"`
18 | }
19 |
20 | type State struct {
21 | TTL int `json:"ttl"`
22 | Authorizations []Authorization `json:"authorizations"`
23 | Identity string `json:"identity"`
24 | IdentityURL string `json:"identity_url"`
25 | Expires time.Time
26 | }
27 |
28 | func (a *Authorization) HasPermission(permission string) bool {
29 | for _, p := range a.Permissions {
30 | if permission == p {
31 | return true
32 | }
33 | }
34 | return false
35 | }
36 |
37 | func (a *Authorization) IsAllowed(topic, channel string) bool {
38 | if channel != "" {
39 | if !a.HasPermission("subscribe") {
40 | return false
41 | }
42 | } else {
43 | if !a.HasPermission("publish") {
44 | return false
45 | }
46 | }
47 |
48 | topicRegex := regexp.MustCompile(a.Topic)
49 |
50 | if !topicRegex.MatchString(topic) {
51 | return false
52 | }
53 |
54 | for _, c := range a.Channels {
55 | channelRegex := regexp.MustCompile(c)
56 | if channelRegex.MatchString(channel) {
57 | return true
58 | }
59 | }
60 | return false
61 | }
62 |
63 | func (a *State) IsAllowed(topic, channel string) bool {
64 | for _, aa := range a.Authorizations {
65 | if aa.IsAllowed(topic, channel) {
66 | return true
67 | }
68 | }
69 | return false
70 | }
71 |
72 | func (a *State) IsExpired() bool {
73 | if a.Expires.Before(time.Now()) {
74 | return true
75 | }
76 | return false
77 | }
78 |
79 | func QueryAnyAuthd(authd []string, remoteIP, tlsEnabled, authSecret string) (*State, error) {
80 | for _, a := range authd {
81 | authState, err := QueryAuthd(a, remoteIP, tlsEnabled, authSecret)
82 | if err != nil {
83 | log.Printf("Error: failed auth against %s %s", a, err)
84 | continue
85 | }
86 | return authState, nil
87 | }
88 | return nil, errors.New("Unable to access auth server")
89 | }
90 |
91 | func QueryAuthd(authd, remoteIP, tlsEnabled, authSecret string) (*State, error) {
92 | v := url.Values{}
93 | v.Set("remote_ip", remoteIP)
94 | v.Set("tls", tlsEnabled)
95 | v.Set("secret", authSecret)
96 |
97 | endpoint := fmt.Sprintf("http://%s/auth?%s", authd, v.Encode())
98 |
99 | var authState State
100 | client := http_api.NewClient(nil)
101 | if _, err := client.GETV1(endpoint, &authState); err != nil {
102 | return nil, err
103 | }
104 |
105 | // validation on response
106 | for _, auth := range authState.Authorizations {
107 | for _, p := range auth.Permissions {
108 | switch p {
109 | case "subscribe", "publish":
110 | default:
111 | return nil, fmt.Errorf("unknown permission %s", p)
112 | }
113 | }
114 |
115 | if _, err := regexp.Compile(auth.Topic); err != nil {
116 | return nil, fmt.Errorf("unable to compile topic %q %s", auth.Topic, err)
117 | }
118 |
119 | for _, channel := range auth.Channels {
120 | if _, err := regexp.Compile(channel); err != nil {
121 | return nil, fmt.Errorf("unable to compile channel %q %s", channel, err)
122 | }
123 | }
124 | }
125 |
126 | if authState.TTL <= 0 {
127 | return nil, fmt.Errorf("invalid TTL %d (must be >0)", authState.TTL)
128 | }
129 |
130 | authState.Expires = time.Now().Add(time.Duration(authState.TTL) * time.Second)
131 | return &authState, nil
132 | }
133 |
--------------------------------------------------------------------------------
/Gopkg.toml:
--------------------------------------------------------------------------------
1 | # Gopkg.toml example
2 | #
3 | # Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
4 | # for detailed Gopkg.toml documentation.
5 | #
6 | # required = ["github.com/user/thing/cmd/thing"]
7 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
8 | #
9 | # [[constraint]]
10 | # name = "github.com/user/project"
11 | # version = "1.0.0"
12 | #
13 | # [[constraint]]
14 | # name = "github.com/user/project2"
15 | # branch = "dev"
16 | # source = "github.com/myfork/project2"
17 | #
18 | # [[override]]
19 | # name = "github.com/x/y"
20 | # version = "2.4.0"
21 | #
22 | # [prune]
23 | # non-go = false
24 | # go-tests = true
25 | # unused-packages = true
26 |
27 |
28 | [[constraint]]
29 | name = "github.com/BurntSushi/toml"
30 | version = "0.3.1"
31 |
32 | [[constraint]]
33 | name = "github.com/Workiva/go-datastructures"
34 | version = "1.0.50"
35 |
36 | [[constraint]]
37 | name = "github.com/absolute8511/bolt"
38 | version = "1.5.0"
39 |
40 | [[constraint]]
41 | name = "github.com/absolute8511/glog"
42 | version = "0.2.0"
43 |
44 | [[constraint]]
45 | branch = "master"
46 | name = "github.com/absolute8511/gorpc"
47 |
48 | [[constraint]]
49 | branch = "master"
50 | name = "github.com/absolute8511/goskiplist"
51 |
52 | [[constraint]]
53 | name = "github.com/astaxie/beego"
54 | version = "1.11.1"
55 |
56 | [[constraint]]
57 | branch = "master"
58 | name = "github.com/bitly/go-hostpool"
59 |
60 | [[constraint]]
61 | name = "github.com/bitly/go-simplejson"
62 | version = "0.5.0"
63 |
64 | [[constraint]]
65 | branch = "master"
66 | name = "github.com/bitly/timer_metrics"
67 |
68 | [[constraint]]
69 | name = "github.com/blang/semver"
70 | version = "3.5.1"
71 |
72 | [[constraint]]
73 | branch = "master"
74 | name = "github.com/bmizerany/perks"
75 |
76 | [[constraint]]
77 | name = "github.com/cenkalti/backoff"
78 | version = "2.1.0"
79 |
80 | [[constraint]]
81 | name = "github.com/coreos/etcd"
82 | version = "2.3.8"
83 |
84 | [[constraint]]
85 | name = "github.com/gobwas/glob"
86 | version = "0.2.3"
87 |
88 | [[constraint]]
89 | name = "github.com/golang/protobuf"
90 | version = "1.2.0"
91 |
92 | [[constraint]]
93 | branch = "master"
94 | name = "github.com/golang/snappy"
95 |
96 | [[constraint]]
97 | name = "github.com/gorilla/sessions"
98 | version = "1.1.3"
99 |
100 | [[constraint]]
101 | name = "github.com/judwhite/go-svc"
102 | version = "1.0.0"
103 |
104 | [[constraint]]
105 | name = "github.com/julienschmidt/httprouter"
106 | version = "1.2.0"
107 |
108 | [[constraint]]
109 | branch = "master"
110 | name = "github.com/mreiferson/go-options"
111 |
112 | [[constraint]]
113 | name = "github.com/spaolacci/murmur3"
114 | version = "1.1.0"
115 |
116 | [[constraint]]
117 | name = "github.com/tidwall/gjson"
118 | version = "1.1.3"
119 |
120 | [[constraint]]
121 | name = "github.com/twinj/uuid"
122 | version = "1.0.0"
123 |
124 | [[constraint]]
125 | branch = "master"
126 | name = "github.com/viki-org/dnscache"
127 |
128 |
129 | [[constraint]]
130 | name = "github.com/hashicorp/golang-lru"
131 | version = "0.5.3"
132 |
133 | [[constraint]]
134 | name = "github.com/youzan/go-nsq"
135 | version = "1.6.1-HA"
136 |
137 | [[constraint]]
138 | branch = "master"
139 | name = "golang.org/x/net"
140 |
141 | [[constraint]]
142 | branch = "master"
143 | name = "golang.org/x/sync"
144 |
145 | [[constraint]]
146 | name = "google.golang.org/grpc"
147 | version = "1.17.0"
148 |
149 | [prune]
150 | go-tests = true
151 | unused-packages = true
152 |
--------------------------------------------------------------------------------
/nsqadmin/static/js/views/topic.js:
--------------------------------------------------------------------------------
1 | var $ = require('jquery');
2 |
3 | window.jQuery = $;
4 | var bootstrap = require('bootstrap'); //eslint-disable-line no-unused-vars
5 | var bootbox = require('bootbox');
6 |
7 | var Pubsub = require('../lib/pubsub');
8 | var AppState = require('../app_state');
9 |
10 | var BaseView = require('./base');
11 |
12 | var click2Show=" >>>";
13 | var click2Hide=" <<<";
14 |
15 | var TopicView = BaseView.extend({
16 | className: 'topic container-fluid',
17 |
18 | template: require('./spinner.hbs'),
19 |
20 | events: {
21 | 'click .topic-actions button': 'topicAction',
22 | 'click .channel-action .hierarchy button': 'onCreateTopicChannel',
23 | 'click .toggle h4': 'onToggle',
24 | 'click .toggle h4 span a': 'onToggle',
25 | },
26 |
27 | initialize: function() {
28 | BaseView.prototype.initialize.apply(this, arguments);
29 | this.listenTo(AppState, 'change:graph_interval', this.render);
30 | this.model.fetch()
31 | .done(function(data) {
32 | this.template = require('./topic.hbs');
33 | this.render({'message': data['message']});
34 | }.bind(this))
35 | .fail(this.handleViewError.bind(this))
36 | .always(Pubsub.trigger.bind(Pubsub, 'view:ready'));
37 | $('[data-toggle="tooltip"]').tooltip({
38 | placement : 'top'
39 | });
40 | },
41 |
42 | onToggle: function(e) {
43 | e.preventDefault();
44 | e.stopPropagation();
45 | var canHideClass = "canHide";
46 | var parent = $(e.target).parents(".toggle").first();
47 | var divCanHide = parent.next();
48 | if(divCanHide != null && divCanHide.attr("class") != null && divCanHide.attr("class").indexOf(canHideClass) !== -1) {
49 | divCanHide.toggle(300);
50 | var anchor = parent.find("a").first();
51 | if(anchor.text().indexOf(click2Hide) !== -1) {
52 | anchor.text(click2Show);
53 | } else {
54 | anchor.text(click2Hide);
55 | }
56 | }
57 | },
58 |
59 | topicAction: function(e) {
60 | e.preventDefault();
61 | e.stopPropagation();
62 | var action = $(e.currentTarget).data('action');
63 | var txt = 'Are you sure you want to ' +
64 | action + ' ' + this.model.get('name') + ' ?';
65 | bootbox.confirm(txt, function(result) {
66 | if (result !== true) {
67 | return;
68 | }
69 | if (action === 'delete') {
70 | $.ajax(this.model.url(), {'method': 'DELETE'})
71 | .done(function() { window.location = '/'; });
72 | } else {
73 | $.post(this.model.url(), JSON.stringify({'action': action}))
74 | .done(function() { window.location.reload(true); })
75 | .fail(this.handleAJAXError.bind(this));
76 | }
77 | }.bind(this));
78 | },
79 |
80 | onCreateTopicChannel: function(e) {
81 | e.preventDefault();
82 | e.stopPropagation();
83 | var topic = $(e.target.form.elements['topic']).val();
84 | var channel = $(e.target.form.elements['channel']).val();
85 | if (topic === '' || channel === '') {
86 | return;
87 | }
88 | $.post(AppState.url('/topics/' + topic + '/' + channel), JSON.stringify({
89 | 'action': 'create'
90 | }))
91 | .done(function() { window.location.reload(true); })
92 | .fail(this.handleAJAXError.bind(this));
93 | }
94 | });
95 |
96 | module.exports = TopicView;
97 |
--------------------------------------------------------------------------------
/nsqadmin/notify.go:
--------------------------------------------------------------------------------
1 | package nsqadmin
2 |
3 | import (
4 | "encoding/base64"
5 | "net/http"
6 | "net/url"
7 | "os"
8 | "strings"
9 | "time"
10 | "encoding/json"
11 | )
12 |
13 | type AdminAction struct {
14 | Action string `json:"action"`
15 | Topic string `json:"topic"`
16 | Order bool `json:"order"`
17 | Channel string `json:"channel,omitempty"`
18 | Node string `json:"node,omitempty"`
19 | Timestamp int64 `json:"timestamp"`
20 | User string `json:"user,omitempty"`
21 | RemoteIP string `json:"remote_ip"`
22 | UserAgent string `json:"user_agent"`
23 | URL string `json:"url"` // The URL of the HTTP request that triggered this action
24 | Via string `json:"via"` // the Hostname of the nsqadmin performing this action
25 | }
26 |
27 | func (a *AdminAction) String() string {
28 | bytes, err := json.Marshal(*a)
29 | if err == nil {
30 | return string(bytes)
31 | }
32 | return ""
33 | }
34 |
35 | func basicAuthUser(req *http.Request) string {
36 | s := strings.SplitN(req.Header.Get("Authorization"), " ", 2)
37 | if len(s) != 2 || s[0] != "Basic" {
38 | return ""
39 | }
40 | b, err := base64.StdEncoding.DecodeString(s[1])
41 | if err != nil {
42 | return ""
43 | }
44 | pair := strings.SplitN(string(b), ":", 2)
45 | if len(pair) != 2 {
46 | return ""
47 | }
48 | return pair[0]
49 | }
50 |
51 |
52 | func (s *httpServer) notifyAdminActionWithUser(action, topic, channel, node string, req *http.Request) {
53 | s.notifyAdminActionWithUserAndOrder(action, topic, channel, node, false, req)
54 | }
55 |
56 | func (s *httpServer) notifyAdminActionWithUserAndOrder(action, topic, channel, node string, order bool, req *http.Request) {
57 | via, _ := os.Hostname()
58 | u := url.URL{
59 | Scheme: "http",
60 | Host: req.Host,
61 | Path: req.URL.Path,
62 | RawQuery: req.URL.RawQuery,
63 | }
64 | if req.TLS != nil || req.Header.Get("X-Scheme") == "https" {
65 | u.Scheme = "https"
66 | }
67 | a := &AdminAction{
68 | Action: action,
69 | Topic: topic,
70 | Order: order,
71 | Channel: channel,
72 | Node: node,
73 | Timestamp: time.Now().Unix(),
74 | RemoteIP: req.RemoteAddr,
75 | UserAgent: req.UserAgent(),
76 | URL: u.String(),
77 | Via: via,
78 | }
79 | user, _ := s.getExistingUserInfo(req)
80 | if user != nil && user.IsLogin() {
81 | a.User = user.GetUserName()
82 | } else {
83 | a.User = basicAuthUser(req)
84 | }
85 | // access log
86 | s.ctx.nsqadmin.logf("ACCESS: %v", a.String())
87 | if s.ctx.nsqadmin.opts.NotificationHTTPEndpoint == "" {
88 | return
89 | }
90 |
91 | // Perform all work in a new goroutine so this never blocks
92 | go func() { s.ctx.nsqadmin.notifications <- a }()
93 | }
94 |
95 | func (s *httpServer) notifyAdminAction(action, topic, channel, node string, req *http.Request) {
96 | if s.ctx.nsqadmin.opts.NotificationHTTPEndpoint == "" {
97 | return
98 | }
99 | via, _ := os.Hostname()
100 |
101 | u := url.URL{
102 | Scheme: "http",
103 | Host: req.Host,
104 | Path: req.URL.Path,
105 | RawQuery: req.URL.RawQuery,
106 | }
107 | if req.TLS != nil || req.Header.Get("X-Scheme") == "https" {
108 | u.Scheme = "https"
109 | }
110 |
111 | a := &AdminAction{
112 | Action: action,
113 | Topic: topic,
114 | Channel: channel,
115 | Node: node,
116 | Timestamp: time.Now().Unix(),
117 | User: basicAuthUser(req),
118 | RemoteIP: req.RemoteAddr,
119 | UserAgent: req.UserAgent(),
120 | URL: u.String(),
121 | Via: via,
122 | }
123 | // Perform all work in a new goroutine so this never blocks
124 | go func() { s.ctx.nsqadmin.notifications <- a }()
125 | }
126 |
--------------------------------------------------------------------------------